drivers/usb/gadget/net2280.c

   1 /*
   2  * Driver for the PLX NET2280 USB device controller.
   3  * Specs and errata are available from <http://www.plxtech.com>.
   4  *
   5  * PLX Technology Inc. (formerly NetChip Technology) supported the
   6  * development of this driver.
   7  *
   8  *
   9  * CODE STATUS HIGHLIGHTS
  10  *
  11  * This driver should work well with most "gadget" drivers, including
  12  * the Mass Storage, Serial, and Ethernet/RNDIS gadget drivers
  13  * as well as Gadget Zero and Gadgetfs.
  14  *
  15  * DMA is enabled by default.  Drivers using transfer queues might use
  16  * DMA chaining to remove IRQ latencies between transfers.  (Except when
  17  * short OUT transfers happen.)  Drivers can use the req->no_interrupt
  18  * hint to completely eliminate some IRQs, if a later IRQ is guaranteed
  19  * and DMA chaining is enabled.
  20  *
  21  * MSI is enabled by default.  The legacy IRQ is used if MSI couldn't
  22  * be enabled.
  23  *
  24  * Note that almost all the errata workarounds here are only needed for
  25  * rev1 chips.  Rev1a silicon (0110) fixes almost all of them.
  26  */
  27
  28 /*
  29  * Copyright (C) 2003 David Brownell
  30  * Copyright (C) 2003-2005 PLX Technology, Inc.
  31  * Copyright (C) 2014 Ricardo Ribalda - Qtechnology/AS
  32  *
  33  * Modified Seth Levy 2005 PLX Technology, Inc. to provide compatibility
  34  *      with 2282 chip
  35  *
  36  * Modified Ricardo Ribalda Qtechnology AS  to provide compatibility
  37  *      with usb 338x chip. Based on PLX driver
  38  *
  39  * This program is free software; you can redistribute it and/or modify
  40  * it under the terms of the GNU General Public License as published by
  41  * the Free Software Foundation; either version 2 of the License, or
  42  * (at your option) any later version.
  43  */
  44
  45 #undef  DEBUG           /* messages on error and most fault paths */
  46 #undef  VERBOSE         /* extra debug messages (success too) */
  47
  48 #include <linux/module.h>
  49 #include <linux/pci.h>
  50 #include <linux/dma-mapping.h>
  51 #include <linux/kernel.h>
  52 #include <linux/delay.h>
  53 #include <linux/ioport.h>
  54 #include <linux/slab.h>
  55 #include <linux/errno.h>
  56 #include <linux/init.h>
  57 #include <linux/timer.h>
  58 #include <linux/list.h>
  59 #include <linux/interrupt.h>
  60 #include <linux/moduleparam.h>
  61 #include <linux/device.h>
  62 #include <linux/usb/ch9.h>
  63 #include <linux/usb/gadget.h>
  64 #include <linux/prefetch.h>
  65
  66 #include <asm/byteorder.h>
  67 #include <asm/io.h>
  68 #include <asm/irq.h>
  69 #include <asm/unaligned.h>
  70
  71 #define DRIVER_DESC             "PLX NET228x/USB338x USB Peripheral Controller"
  72 #define DRIVER_VERSION          "2005 Sept 27/v3.0"
  73
  74 #define EP_DONTUSE              13      /* nonzero */
  75
  76 #define USE_RDK_LEDS            /* GPIO pins control three LEDs */
  77
  78
  79 static const char driver_name [] = "net2280";
  80 static const char driver_desc [] = DRIVER_DESC;
  81
  82 static const u32 ep_bit[9] = { 0, 17, 2, 19, 4, 1, 18, 3, 20 };
  83 static const char ep0name [] = "ep0";
  84 static const char *const ep_name [] = {
  85         ep0name,
  86         "ep-a", "ep-b", "ep-c", "ep-d",
  87         "ep-e", "ep-f", "ep-g", "ep-h",
  88 };
  89
  90 /* use_dma -- general goodness, fewer interrupts, less cpu load (vs PIO)
  91  * use_dma_chaining -- dma descriptor queueing gives even more irq reduction
  92  *
  93  * The net2280 DMA engines are not tightly integrated with their FIFOs;
  94  * not all cases are (yet) handled well in this driver or the silicon.
  95  * Some gadget drivers work better with the dma support here than others.
  96  * These two parameters let you use PIO or more aggressive DMA.
  97  */
  98 static bool use_dma = 1;
  99 static bool use_dma_chaining = 0;
 100 static bool use_msi = 1;
 101
 102 /* "modprobe net2280 use_dma=n" etc */
 103 module_param (use_dma, bool, S_IRUGO);
 104 module_param (use_dma_chaining, bool, S_IRUGO);
 105 module_param(use_msi, bool, S_IRUGO);
 106
 107 /* mode 0 == ep-{a,b,c,d} 1K fifo each
 108  * mode 1 == ep-{a,b} 2K fifo each, ep-{c,d} unavailable
 109  * mode 2 == ep-a 2K fifo, ep-{b,c} 1K each, ep-d unavailable
 110  */
 111 static ushort fifo_mode = 0;
 112
 113 /* "modprobe net2280 fifo_mode=1" etc */
 114 module_param (fifo_mode, ushort, 0644);
 115
 116 /* enable_suspend -- When enabled, the driver will respond to
 117  * USB suspend requests by powering down the NET2280.  Otherwise,
 118  * USB suspend requests will be ignored.  This is acceptable for
 119  * self-powered devices
 120  */
 121 static bool enable_suspend = 0;
 122
 123 /* "modprobe net2280 enable_suspend=1" etc */
 124 module_param (enable_suspend, bool, S_IRUGO);
 125
 126 /* force full-speed operation */
 127 static bool full_speed;
 128 module_param(full_speed, bool, 0444);
 129 MODULE_PARM_DESC(full_speed, "force full-speed mode -- for testing only!");
 130
 131 #define DIR_STRING(bAddress) (((bAddress) & USB_DIR_IN) ? "in" : "out")
 132
 133 #if defined(CONFIG_USB_GADGET_DEBUG_FILES) || defined (DEBUG)
 134 static char *type_string (u8 bmAttributes)
 135 {
 136         switch ((bmAttributes) & USB_ENDPOINT_XFERTYPE_MASK) {
 137         case USB_ENDPOINT_XFER_BULK:    return "bulk";
 138         case USB_ENDPOINT_XFER_ISOC:    return "iso";
 139         case USB_ENDPOINT_XFER_INT:     return "intr";
 140         }
 141         return "control";
 142 }
 143 #endif
 144
 145 #include "net2280.h"
 146
 147 #define valid_bit       cpu_to_le32 (1 << VALID_BIT)
 148 #define dma_done_ie     cpu_to_le32 (1 << DMA_DONE_INTERRUPT_ENABLE)
 149
 150 /*-------------------------------------------------------------------------*/
 151
 152 static int
 153 net2280_enable (struct usb_ep *_ep, const struct usb_endpoint_descriptor *desc)
 154 {
 155         struct net2280          *dev;
 156         struct net2280_ep       *ep;
 157         u32                     max, tmp;
 158         unsigned long           flags;
 159         static const u32 ep_key[9] = { 1, 0, 1, 0, 1, 1, 0, 1, 0 };
 160         static const u32 ep_enhanced[9] = { 0x10, 0x60, 0x30, 0x80,
 161                                           0x50, 0x20, 0x70, 0x40, 0x90 };
 162
 163         ep = container_of (_ep, struct net2280_ep, ep);
 164         if (!_ep || !desc || ep->desc || _ep->name == ep0name
 165                         || desc->bDescriptorType != USB_DT_ENDPOINT)
 166                 return -EINVAL;
 167         dev = ep->dev;
 168         if (!dev->driver || dev->gadget.speed == USB_SPEED_UNKNOWN)
 169                 return -ESHUTDOWN;
 170
 171         /* erratum 0119 workaround ties up an endpoint number */
 172         if ((desc->bEndpointAddress & 0x0f) == EP_DONTUSE)
 173                 return -EDOM;
 174
 175         if (dev->pdev->vendor == 0x10b5) {
 176                 if ((desc->bEndpointAddress & 0x0f) >= 0x0c)
 177                         return -EDOM;
 178                 ep->is_in = !!usb_endpoint_dir_in(desc);
 179                 if (dev->enhanced_mode && ep->is_in && ep_key[ep->num])
 180                         return -EINVAL;
 181         }
 182
 183         /* sanity check ep-e/ep-f since their fifos are small */
 184         max = usb_endpoint_maxp (desc) & 0x1fff;
 185         if (ep->num > 4 && max > 64 && (dev->pdev->vendor == 0x17cc))
 186                 return -ERANGE;
 187
 188
 189         spin_lock_irqsave (&dev->lock, flags);
 190         _ep->maxpacket = max & 0x7ff;
 191         ep->desc = desc;
 192
 193         /* ep_reset() has already been called */
 194         ep->stopped = 0;
 195         ep->wedged = 0;
 196         ep->out_overflow = 0;
 197
 198         /* set speed-dependent max packet; may kick in high bandwidth */
 199         set_idx_reg(dev->regs, (dev->enhanced_mode) ? ep_enhanced[ep->num]
 200                                         : REG_EP_MAXPKT(dev, ep->num), max);
 201
 202         /* FIFO lines can't go to different packets.  PIO is ok, so
 203          * use it instead of troublesome (non-bulk) multi-packet DMA.
 204          */
 205         if (ep->dma && (max % 4) != 0 && use_dma_chaining) {
 206                 DEBUG (ep->dev, "%s, no dma for maxpacket %d\n",
 207                         ep->ep.name, ep->ep.maxpacket);
 208                 ep->dma = NULL;
 209         }
 210
 211         /* set type, direction, address; reset fifo counters */
 212         writel ((1 << FIFO_FLUSH), &ep->regs->ep_stat);
 213         tmp = (desc->bmAttributes & USB_ENDPOINT_XFERTYPE_MASK);
 214         if (tmp == USB_ENDPOINT_XFER_INT) {
 215                 /* erratum 0105 workaround prevents hs NYET */
 216                 if (dev->chiprev == 0100
 217                                 && dev->gadget.speed == USB_SPEED_HIGH
 218                                 && !(desc->bEndpointAddress & USB_DIR_IN))
 219                         writel ((1 << CLEAR_NAK_OUT_PACKETS_MODE),
 220                                 &ep->regs->ep_rsp);
 221         } else if (tmp == USB_ENDPOINT_XFER_BULK) {
 222                 /* catch some particularly blatant driver bugs */
 223                 if ((dev->gadget.speed == USB_SPEED_SUPER && max != 1024) ||
 224                     (dev->gadget.speed == USB_SPEED_HIGH && max != 512) ||
 225                     (dev->gadget.speed == USB_SPEED_FULL && max > 64)) {
 226                         spin_unlock_irqrestore(&dev->lock, flags);
 227                         return -ERANGE;
 228                 }
 229         }
 230         ep->is_iso = (tmp == USB_ENDPOINT_XFER_ISOC) ? 1 : 0;
 231         /* Enable this endpoint */
 232         if (dev->pdev->vendor == 0x17cc) {
 233                 tmp <<= ENDPOINT_TYPE;
 234                 tmp |= desc->bEndpointAddress;
 235                 /* default full fifo lines */
 236                 tmp |= (4 << ENDPOINT_BYTE_COUNT);
 237                 tmp |= 1 << ENDPOINT_ENABLE;
 238                 ep->is_in = (tmp & USB_DIR_IN) != 0;
 239         } else {
 240                 /* In Legacy mode, only OUT endpoints are used */
 241                 if (dev->enhanced_mode && ep->is_in) {
 242                         tmp <<= IN_ENDPOINT_TYPE;
 243                         tmp |= (1 << IN_ENDPOINT_ENABLE);
 244                         /* Not applicable to Legacy */
 245                         tmp |= (1 << ENDPOINT_DIRECTION);
 246                 } else {
 247                         tmp <<= OUT_ENDPOINT_TYPE;
 248                         tmp |= (1 << OUT_ENDPOINT_ENABLE);
 249                         tmp |= (ep->is_in << ENDPOINT_DIRECTION);
 250                 }
 251
 252                 tmp |= usb_endpoint_num(desc);
 253                 tmp |= (ep->ep.maxburst << MAX_BURST_SIZE);
 254         }
 255
 256         /* Make sure all the registers are written before ep_rsp*/
 257         wmb();
 258
 259         /* for OUT transfers, block the rx fifo until a read is posted */
 260         if (!ep->is_in)
 261                 writel ((1 << SET_NAK_OUT_PACKETS), &ep->regs->ep_rsp);
 262         else if (dev->pdev->device != 0x2280) {
 263                 /* Added for 2282, Don't use nak packets on an in endpoint,
 264                  * this was ignored on 2280
 265                  */
 266                 writel ((1 << CLEAR_NAK_OUT_PACKETS)
 267                         | (1 << CLEAR_NAK_OUT_PACKETS_MODE), &ep->regs->ep_rsp);
 268         }
 269
 270         writel(tmp, &ep->cfg->ep_cfg);
 271
 272         /* enable irqs */
 273         if (!ep->dma) {                         /* pio, per-packet */
 274                 tmp = (dev->pdev->vendor == 0x17cc)?(1 << ep->num)
 275                                                    : (1 << ep_bit[ep->num]);
 276                 tmp |= readl(&dev->regs->pciirqenb0);
 277                 writel (tmp, &dev->regs->pciirqenb0);
 278
 279                 tmp = (1 << DATA_PACKET_RECEIVED_INTERRUPT_ENABLE)
 280                         | (1 << DATA_PACKET_TRANSMITTED_INTERRUPT_ENABLE);
 281                 if (dev->pdev->device == 0x2280)
 282                         tmp |= readl (&ep->regs->ep_irqenb);
 283                 writel (tmp, &ep->regs->ep_irqenb);
 284         } else {                                /* dma, per-request */
 285                 tmp = (1 << (8 + ep->num));     /* completion */
 286                 tmp |= readl (&dev->regs->pciirqenb1);
 287                 writel (tmp, &dev->regs->pciirqenb1);
 288
 289                 /* for short OUT transfers, dma completions can't
 290                  * advance the queue; do it pio-style, by hand.
 291                  * NOTE erratum 0112 workaround #2
 292                  */
 293                 if ((desc->bEndpointAddress & USB_DIR_IN) == 0) {
 294                         tmp = (1 << SHORT_PACKET_TRANSFERRED_INTERRUPT_ENABLE);
 295                         writel (tmp, &ep->regs->ep_irqenb);
 296
 297                         tmp = (dev->pdev->vendor == 0x17cc)?(1 << ep->num)
 298                                                 : (1 << ep_bit[ep->num]);
 299                         tmp |= readl(&dev->regs->pciirqenb0);
 300                         writel(tmp, &dev->regs->pciirqenb0);
 301                 }
 302         }
 303
 304         tmp = desc->bEndpointAddress;
 305         DEBUG (dev, "enabled %s (ep%d%s-%s) %s max %04x\n",
 306                 _ep->name, tmp & 0x0f, DIR_STRING (tmp),
 307                 type_string (desc->bmAttributes),
 308                 ep->dma ? "dma" : "pio", max);
 309
 310         /* pci writes may still be posted */
 311         spin_unlock_irqrestore (&dev->lock, flags);
 312         return 0;
 313 }
 314
 315 static int handshake (u32 __iomem *ptr, u32 mask, u32 done, int usec)
 316 {
 317         u32     result;
 318
 319         do {
 320                 result = readl (ptr);
 321                 if (result == ~(u32)0)          /* "device unplugged" */
 322                         return -ENODEV;
 323                 result &= mask;
 324                 if (result == done)
 325                         return 0;
 326                 udelay (1);
 327                 usec--;
 328         } while (usec > 0);
 329         return -ETIMEDOUT;
 330 }
 331
 332 static const struct usb_ep_ops net2280_ep_ops;
 333
 334 static void ep_reset_228x(struct net2280_regs __iomem *regs,
 335                           struct net2280_ep *ep)
 336 {
 337         u32             tmp;
 338
 339         ep->desc = NULL;
 340         INIT_LIST_HEAD (&ep->queue);
 341
 342         usb_ep_set_maxpacket_limit(&ep->ep, ~0);
 343         ep->ep.ops = &net2280_ep_ops;
 344
 345         /* disable the dma, irqs, endpoint... */
 346         if (ep->dma) {
 347                 writel (0, &ep->dma->dmactl);
 348                 writel (  (1 << DMA_SCATTER_GATHER_DONE_INTERRUPT)
 349                         | (1 << DMA_TRANSACTION_DONE_INTERRUPT)
 350                         | (1 << DMA_ABORT)
 351                         , &ep->dma->dmastat);
 352
 353                 tmp = readl (&regs->pciirqenb0);
 354                 tmp &= ~(1 << ep->num);
 355                 writel (tmp, &regs->pciirqenb0);
 356         } else {
 357                 tmp = readl (&regs->pciirqenb1);
 358                 tmp &= ~(1 << (8 + ep->num));   /* completion */
 359                 writel (tmp, &regs->pciirqenb1);
 360         }
 361         writel (0, &ep->regs->ep_irqenb);
 362
 363         /* init to our chosen defaults, notably so that we NAK OUT
 364          * packets until the driver queues a read (+note erratum 0112)
 365          */
 366         if (!ep->is_in || ep->dev->pdev->device == 0x2280) {
 367                 tmp = (1 << SET_NAK_OUT_PACKETS_MODE)
 368                 | (1 << SET_NAK_OUT_PACKETS)
 369                 | (1 << CLEAR_EP_HIDE_STATUS_PHASE)
 370                 | (1 << CLEAR_INTERRUPT_MODE);
 371         } else {
 372                 /* added for 2282 */
 373                 tmp = (1 << CLEAR_NAK_OUT_PACKETS_MODE)
 374                 | (1 << CLEAR_NAK_OUT_PACKETS)
 375                 | (1 << CLEAR_EP_HIDE_STATUS_PHASE)
 376                 | (1 << CLEAR_INTERRUPT_MODE);
 377         }
 378
 379         if (ep->num != 0) {
 380                 tmp |= (1 << CLEAR_ENDPOINT_TOGGLE)
 381                         | (1 << CLEAR_ENDPOINT_HALT);
 382         }
 383         writel (tmp, &ep->regs->ep_rsp);
 384
 385         /* scrub most status bits, and flush any fifo state */
 386         if (ep->dev->pdev->device == 0x2280)
 387                 tmp = (1 << FIFO_OVERFLOW)
 388                         | (1 << FIFO_UNDERFLOW);
 389         else
 390                 tmp = 0;
 391
 392         writel (tmp | (1 << TIMEOUT)
 393                 | (1 << USB_STALL_SENT)
 394                 | (1 << USB_IN_NAK_SENT)
 395                 | (1 << USB_IN_ACK_RCVD)
 396                 | (1 << USB_OUT_PING_NAK_SENT)
 397                 | (1 << USB_OUT_ACK_SENT)
 398                 | (1 << FIFO_FLUSH)
 399                 | (1 << SHORT_PACKET_OUT_DONE_INTERRUPT)
 400                 | (1 << SHORT_PACKET_TRANSFERRED_INTERRUPT)
 401                 | (1 << DATA_PACKET_RECEIVED_INTERRUPT)
 402                 | (1 << DATA_PACKET_TRANSMITTED_INTERRUPT)
 403                 | (1 << DATA_OUT_PING_TOKEN_INTERRUPT)
 404                 | (1 << DATA_IN_TOKEN_INTERRUPT)
 405                 , &ep->regs->ep_stat);
 406
 407         /* fifo size is handled separately */
 408 }
 409
 410 static void ep_reset_338x(struct net2280_regs __iomem *regs,
 411                                         struct net2280_ep *ep)
 412 {
 413         u32 tmp, dmastat;
 414
 415         ep->desc = NULL;
 416         INIT_LIST_HEAD(&ep->queue);
 417
 418         usb_ep_set_maxpacket_limit(&ep->ep, ~0);
 419         ep->ep.ops = &net2280_ep_ops;
 420
 421         /* disable the dma, irqs, endpoint... */
 422         if (ep->dma) {
 423                 writel(0, &ep->dma->dmactl);
 424                 writel((1 << DMA_ABORT_DONE_INTERRUPT) |
 425                        (1 << DMA_PAUSE_DONE_INTERRUPT) |
 426                        (1 << DMA_SCATTER_GATHER_DONE_INTERRUPT) |
 427                        (1 << DMA_TRANSACTION_DONE_INTERRUPT)
 428                        /* | (1 << DMA_ABORT) */
 429                        , &ep->dma->dmastat);
 430
 431                 dmastat = readl(&ep->dma->dmastat);
 432                 if (dmastat == 0x5002) {
 433                         WARNING(ep->dev, "The dmastat return = %x!!\n",
 434                                dmastat);
 435                         writel(0x5a, &ep->dma->dmastat);
 436                 }
 437
 438                 tmp = readl(&regs->pciirqenb0);
 439                 tmp &= ~(1 << ep_bit[ep->num]);
 440                 writel(tmp, &regs->pciirqenb0);
 441         } else {
 442                 if (ep->num < 5) {
 443                         tmp = readl(&regs->pciirqenb1);
 444                         tmp &= ~(1 << (8 + ep->num));   /* completion */
 445                         writel(tmp, &regs->pciirqenb1);
 446                 }
 447         }
 448         writel(0, &ep->regs->ep_irqenb);
 449
 450         writel((1 << SHORT_PACKET_OUT_DONE_INTERRUPT) |
 451                (1 << SHORT_PACKET_TRANSFERRED_INTERRUPT) |
 452                (1 << FIFO_OVERFLOW) |
 453                (1 << DATA_PACKET_RECEIVED_INTERRUPT) |
 454                (1 << DATA_PACKET_TRANSMITTED_INTERRUPT) |
 455                (1 << DATA_OUT_PING_TOKEN_INTERRUPT) |
 456                (1 << DATA_IN_TOKEN_INTERRUPT), &ep->regs->ep_stat);
 457 }
 458
 459 static void nuke (struct net2280_ep *);
 460
 461 static int net2280_disable (struct usb_ep *_ep)
 462 {
 463         struct net2280_ep       *ep;
 464         unsigned long           flags;
 465
 466         ep = container_of (_ep, struct net2280_ep, ep);
 467         if (!_ep || !ep->desc || _ep->name == ep0name)
 468                 return -EINVAL;
 469
 470         spin_lock_irqsave (&ep->dev->lock, flags);
 471         nuke (ep);
 472
 473         if (ep->dev->pdev->vendor == 0x10b5)
 474                 ep_reset_338x(ep->dev->regs, ep);
 475         else
 476                 ep_reset_228x(ep->dev->regs, ep);
 477
 478         VDEBUG (ep->dev, "disabled %s %s\n",
 479                         ep->dma ? "dma" : "pio", _ep->name);
 480
 481         /* synch memory views with the device */
 482         (void)readl(&ep->cfg->ep_cfg);
 483
 484         if (use_dma && !ep->dma && ep->num >= 1 && ep->num <= 4)
 485                 ep->dma = &ep->dev->dma [ep->num - 1];
 486
 487         spin_unlock_irqrestore (&ep->dev->lock, flags);
 488         return 0;
 489 }
 490
 491 /*-------------------------------------------------------------------------*/
 492
 493 static struct usb_request *
 494 net2280_alloc_request (struct usb_ep *_ep, gfp_t gfp_flags)
 495 {
 496         struct net2280_ep       *ep;
 497         struct net2280_request  *req;
 498
 499         if (!_ep)
 500                 return NULL;
 501         ep = container_of (_ep, struct net2280_ep, ep);
 502
 503         req = kzalloc(sizeof(*req), gfp_flags);
 504         if (!req)
 505                 return NULL;
 506
 507         INIT_LIST_HEAD (&req->queue);
 508
 509         /* this dma descriptor may be swapped with the previous dummy */
 510         if (ep->dma) {
 511                 struct net2280_dma      *td;
 512
 513                 td = pci_pool_alloc (ep->dev->requests, gfp_flags,
 514                                 &req->td_dma);
 515                 if (!td) {
 516                         kfree (req);
 517                         return NULL;
 518                 }
 519                 td->dmacount = 0;       /* not VALID */
 520                 td->dmadesc = td->dmaaddr;
 521                 req->td = td;
 522         }
 523         return &req->req;
 524 }
 525
 526 static void
 527 net2280_free_request (struct usb_ep *_ep, struct usb_request *_req)
 528 {
 529         struct net2280_ep       *ep;
 530         struct net2280_request  *req;
 531
 532         ep = container_of (_ep, struct net2280_ep, ep);
 533         if (!_ep || !_req)
 534                 return;
 535
 536         req = container_of (_req, struct net2280_request, req);
 537         WARN_ON (!list_empty (&req->queue));
 538         if (req->td)
 539                 pci_pool_free (ep->dev->requests, req->td, req->td_dma);
 540         kfree (req);
 541 }
 542
 543 /*-------------------------------------------------------------------------*/
 544
 545 /* load a packet into the fifo we use for usb IN transfers.
 546  * works for all endpoints.
 547  *
 548  * NOTE: pio with ep-a..ep-d could stuff multiple packets into the fifo
 549  * at a time, but this code is simpler because it knows it only writes
 550  * one packet.  ep-a..ep-d should use dma instead.
 551  */
 552 static void
 553 write_fifo (struct net2280_ep *ep, struct usb_request *req)
 554 {
 555         struct net2280_ep_regs  __iomem *regs = ep->regs;
 556         u8                      *buf;
 557         u32                     tmp;
 558         unsigned                count, total;
 559
 560         /* INVARIANT:  fifo is currently empty. (testable) */
 561
 562         if (req) {
 563                 buf = req->buf + req->actual;
 564                 prefetch (buf);
 565                 total = req->length - req->actual;
 566         } else {
 567                 total = 0;
 568                 buf = NULL;
 569         }
 570
 571         /* write just one packet at a time */
 572         count = ep->ep.maxpacket;
 573         if (count > total)      /* min() cannot be used on a bitfield */
 574                 count = total;
 575
 576         VDEBUG (ep->dev, "write %s fifo (IN) %d bytes%s req %p\n",
 577                         ep->ep.name, count,
 578                         (count != ep->ep.maxpacket) ? " (short)" : "",
 579                         req);
 580         while (count >= 4) {
 581                 /* NOTE be careful if you try to align these. fifo lines
 582                  * should normally be full (4 bytes) and successive partial
 583                  * lines are ok only in certain cases.
 584                  */
 585                 tmp = get_unaligned ((u32 *)buf);
 586                 cpu_to_le32s (&tmp);
 587                 writel (tmp, &regs->ep_data);
 588                 buf += 4;
 589                 count -= 4;
 590         }
 591
 592         /* last fifo entry is "short" unless we wrote a full packet.
 593          * also explicitly validate last word in (periodic) transfers
 594          * when maxpacket is not a multiple of 4 bytes.
 595          */
 596         if (count || total < ep->ep.maxpacket) {
 597                 tmp = count ? get_unaligned ((u32 *)buf) : count;
 598                 cpu_to_le32s (&tmp);
 599                 set_fifo_bytecount (ep, count & 0x03);
 600                 writel (tmp, &regs->ep_data);
 601         }
 602
 603         /* pci writes may still be posted */
 604 }
 605
 606 /* work around erratum 0106: PCI and USB race over the OUT fifo.
 607  * caller guarantees chiprev 0100, out endpoint is NAKing, and
 608  * there's no real data in the fifo.
 609  *
 610  * NOTE:  also used in cases where that erratum doesn't apply:
 611  * where the host wrote "too much" data to us.
 612  */
 613 static void out_flush (struct net2280_ep *ep)
 614 {
 615         u32     __iomem *statp;
 616         u32     tmp;
 617
 618         ASSERT_OUT_NAKING (ep);
 619
 620         statp = &ep->regs->ep_stat;
 621         writel (  (1 << DATA_OUT_PING_TOKEN_INTERRUPT)
 622                 | (1 << DATA_PACKET_RECEIVED_INTERRUPT)
 623                 , statp);
 624         writel ((1 << FIFO_FLUSH), statp);
 625         mb ();
 626         tmp = readl (statp);
 627         if (tmp & (1 << DATA_OUT_PING_TOKEN_INTERRUPT)
 628                         /* high speed did bulk NYET; fifo isn't filling */
 629                         && ep->dev->gadget.speed == USB_SPEED_FULL) {
 630                 unsigned        usec;
 631
 632                 usec = 50;              /* 64 byte bulk/interrupt */
 633                 handshake (statp, (1 << USB_OUT_PING_NAK_SENT),
 634                                 (1 << USB_OUT_PING_NAK_SENT), usec);
 635                 /* NAK done; now CLEAR_NAK_OUT_PACKETS is safe */
 636         }
 637 }
 638
 639 /* unload packet(s) from the fifo we use for usb OUT transfers.
 640  * returns true iff the request completed, because of short packet
 641  * or the request buffer having filled with full packets.
 642  *
 643  * for ep-a..ep-d this will read multiple packets out when they
 644  * have been accepted.
 645  */
 646 static int
 647 read_fifo (struct net2280_ep *ep, struct net2280_request *req)
 648 {
 649         struct net2280_ep_regs  __iomem *regs = ep->regs;
 650         u8                      *buf = req->req.buf + req->req.actual;
 651         unsigned                count, tmp, is_short;
 652         unsigned                cleanup = 0, prevent = 0;
 653
 654         /* erratum 0106 ... packets coming in during fifo reads might
 655          * be incompletely rejected.  not all cases have workarounds.
 656          */
 657         if (ep->dev->chiprev == 0x0100
 658                         && ep->dev->gadget.speed == USB_SPEED_FULL) {
 659                 udelay (1);
 660                 tmp = readl (&ep->regs->ep_stat);
 661                 if ((tmp & (1 << NAK_OUT_PACKETS)))
 662                         cleanup = 1;
 663                 else if ((tmp & (1 << FIFO_FULL))) {
 664                         start_out_naking (ep);
 665                         prevent = 1;
 666                 }
 667                 /* else: hope we don't see the problem */
 668         }
 669
 670         /* never overflow the rx buffer. the fifo reads packets until
 671          * it sees a short one; we might not be ready for them all.
 672          */
 673         prefetchw (buf);
 674         count = readl (&regs->ep_avail);
 675         if (unlikely (count == 0)) {
 676                 udelay (1);
 677                 tmp = readl (&ep->regs->ep_stat);
 678                 count = readl (&regs->ep_avail);
 679                 /* handled that data already? */
 680                 if (count == 0 && (tmp & (1 << NAK_OUT_PACKETS)) == 0)
 681                         return 0;
 682         }
 683
 684         tmp = req->req.length - req->req.actual;
 685         if (count > tmp) {
 686                 /* as with DMA, data overflow gets flushed */
 687                 if ((tmp % ep->ep.maxpacket) != 0) {
 688                         ERROR (ep->dev,
 689                                 "%s out fifo %d bytes, expected %d\n",
 690                                 ep->ep.name, count, tmp);
 691                         req->req.status = -EOVERFLOW;
 692                         cleanup = 1;
 693                         /* NAK_OUT_PACKETS will be set, so flushing is safe;
 694                          * the next read will start with the next packet
 695                          */
 696                 } /* else it's a ZLP, no worries */
 697                 count = tmp;
 698         }
 699         req->req.actual += count;
 700
 701         is_short = (count == 0) || ((count % ep->ep.maxpacket) != 0);
 702
 703         VDEBUG (ep->dev, "read %s fifo (OUT) %d bytes%s%s%s req %p %d/%d\n",
 704                         ep->ep.name, count, is_short ? " (short)" : "",
 705                         cleanup ? " flush" : "", prevent ? " nak" : "",
 706                         req, req->req.actual, req->req.length);
 707
 708         while (count >= 4) {
 709                 tmp = readl (&regs->ep_data);
 710                 cpu_to_le32s (&tmp);
 711                 put_unaligned (tmp, (u32 *)buf);
 712                 buf += 4;
 713                 count -= 4;
 714         }
 715         if (count) {
 716                 tmp = readl (&regs->ep_data);
 717                 /* LE conversion is implicit here: */
 718                 do {
 719                         *buf++ = (u8) tmp;
 720                         tmp >>= 8;
 721                 } while (--count);
 722         }
 723         if (cleanup)
 724                 out_flush (ep);
 725         if (prevent) {
 726                 writel ((1 << CLEAR_NAK_OUT_PACKETS), &ep->regs->ep_rsp);
 727                 (void) readl (&ep->regs->ep_rsp);
 728         }
 729
 730         return is_short || ((req->req.actual == req->req.length)
 731                                 && !req->req.zero);
 732 }
 733
 734 /* fill out dma descriptor to match a given request */
 735 static void
 736 fill_dma_desc (struct net2280_ep *ep, struct net2280_request *req, int valid)
 737 {
 738         struct net2280_dma      *td = req->td;
 739         u32                     dmacount = req->req.length;
 740
 741         /* don't let DMA continue after a short OUT packet,
 742          * so overruns can't affect the next transfer.
 743          * in case of overruns on max-size packets, we can't
 744          * stop the fifo from filling but we can flush it.
 745          */
 746         if (ep->is_in)
 747                 dmacount |= (1 << DMA_DIRECTION);
 748         if ((!ep->is_in && (dmacount % ep->ep.maxpacket) != 0)
 749                         || ep->dev->pdev->device != 0x2280)
 750                 dmacount |= (1 << END_OF_CHAIN);
 751
 752         req->valid = valid;
 753         if (valid)
 754                 dmacount |= (1 << VALID_BIT);
 755         if (likely(!req->req.no_interrupt || !use_dma_chaining))
 756                 dmacount |= (1 << DMA_DONE_INTERRUPT_ENABLE);
 757
 758         /* td->dmadesc = previously set by caller */
 759         td->dmaaddr = cpu_to_le32 (req->req.dma);
 760
 761         /* 2280 may be polling VALID_BIT through ep->dma->dmadesc */
 762         wmb ();
 763         td->dmacount = cpu_to_le32(dmacount);
 764 }
 765
 766 static const u32 dmactl_default =
 767                   (1 << DMA_SCATTER_GATHER_DONE_INTERRUPT)
 768                 | (1 << DMA_CLEAR_COUNT_ENABLE)
 769                 /* erratum 0116 workaround part 1 (use POLLING) */
 770                 | (POLL_100_USEC << DESCRIPTOR_POLLING_RATE)
 771                 | (1 << DMA_VALID_BIT_POLLING_ENABLE)
 772                 | (1 << DMA_VALID_BIT_ENABLE)
 773                 | (1 << DMA_SCATTER_GATHER_ENABLE)
 774                 /* erratum 0116 workaround part 2 (no AUTOSTART) */
 775                 | (1 << DMA_ENABLE);
 776
 777 static inline void spin_stop_dma (struct net2280_dma_regs __iomem *dma)
 778 {
 779         handshake (&dma->dmactl, (1 << DMA_ENABLE), 0, 50);
 780 }
 781
 782 static inline void stop_dma (struct net2280_dma_regs __iomem *dma)
 783 {
 784         writel (readl (&dma->dmactl) & ~(1 << DMA_ENABLE), &dma->dmactl);
 785         spin_stop_dma (dma);
 786 }
 787
 788 static void start_queue (struct net2280_ep *ep, u32 dmactl, u32 td_dma)
 789 {
 790         struct net2280_dma_regs __iomem *dma = ep->dma;
 791         unsigned int tmp = (1 << VALID_BIT) | (ep->is_in << DMA_DIRECTION);
 792
 793         if (ep->dev->pdev->device != 0x2280)
 794                 tmp |= (1 << END_OF_CHAIN);
 795
 796         writel (tmp, &dma->dmacount);
 797         writel (readl (&dma->dmastat), &dma->dmastat);
 798
 799         writel (td_dma, &dma->dmadesc);
 800         if (ep->dev->pdev->vendor == 0x10b5)
 801                 dmactl |= (0x01 << DMA_REQUEST_OUTSTANDING);
 802         writel (dmactl, &dma->dmactl);
 803
 804         /* erratum 0116 workaround part 3:  pci arbiter away from net2280 */
 805         (void) readl (&ep->dev->pci->pcimstctl);
 806
 807         writel ((1 << DMA_START), &dma->dmastat);
 808
 809         if (!ep->is_in)
 810                 stop_out_naking (ep);
 811 }
 812
 813 static void start_dma (struct net2280_ep *ep, struct net2280_request *req)
 814 {
 815         u32                     tmp;
 816         struct net2280_dma_regs __iomem *dma = ep->dma;
 817
 818         /* FIXME can't use DMA for ZLPs */
 819
 820         /* on this path we "know" there's no dma active (yet) */
 821         WARN_ON (readl (&dma->dmactl) & (1 << DMA_ENABLE));
 822         writel (0, &ep->dma->dmactl);
 823
 824         /* previous OUT packet might have been short */
 825         if (!ep->is_in && ((tmp = readl (&ep->regs->ep_stat))
 826                                 & (1 << NAK_OUT_PACKETS)) != 0) {
 827                 writel ((1 << SHORT_PACKET_TRANSFERRED_INTERRUPT),
 828                         &ep->regs->ep_stat);
 829
 830                 tmp = readl (&ep->regs->ep_avail);
 831                 if (tmp) {
 832                         writel (readl (&dma->dmastat), &dma->dmastat);
 833
 834                         /* transfer all/some fifo data */
 835                         writel (req->req.dma, &dma->dmaaddr);
 836                         tmp = min (tmp, req->req.length);
 837
 838                         /* dma irq, faking scatterlist status */
 839                         req->td->dmacount = cpu_to_le32 (req->req.length - tmp);
 840                         writel ((1 << DMA_DONE_INTERRUPT_ENABLE)
 841                                 | tmp, &dma->dmacount);
 842                         req->td->dmadesc = 0;
 843                         req->valid = 1;
 844
 845                         writel ((1 << DMA_ENABLE), &dma->dmactl);
 846                         writel ((1 << DMA_START), &dma->dmastat);
 847                         return;
 848                 }
 849         }
 850
 851         tmp = dmactl_default;
 852
 853         /* force packet boundaries between dma requests, but prevent the
 854          * controller from automagically writing a last "short" packet
 855          * (zero length) unless the driver explicitly said to do that.
 856          */
 857         if (ep->is_in) {
 858                 if (likely ((req->req.length % ep->ep.maxpacket) != 0
 859                                 || req->req.zero)) {
 860                         tmp |= (1 << DMA_FIFO_VALIDATE);
 861                         ep->in_fifo_validate = 1;
 862                 } else
 863                         ep->in_fifo_validate = 0;
 864         }
 865
 866         /* init req->td, pointing to the current dummy */
 867         req->td->dmadesc = cpu_to_le32 (ep->td_dma);
 868         fill_dma_desc (ep, req, 1);
 869
 870         if (!use_dma_chaining)
 871                 req->td->dmacount |= cpu_to_le32 (1 << END_OF_CHAIN);
 872
 873         start_queue (ep, tmp, req->td_dma);
 874 }
 875
 876 static inline void resume_dma(struct net2280_ep *ep)
 877 {
 878         writel(readl(&ep->dma->dmactl) | (1 << DMA_ENABLE), &ep->dma->dmactl);
 879
 880         ep->dma_started = true;
 881 }
 882
 883 static inline void ep_stop_dma(struct net2280_ep *ep)
 884 {
 885         writel(readl(&ep->dma->dmactl) & ~(1 << DMA_ENABLE), &ep->dma->dmactl);
 886         spin_stop_dma(ep->dma);
 887
 888         ep->dma_started = false;
 889 }
 890
 891 static inline void
 892 queue_dma (struct net2280_ep *ep, struct net2280_request *req, int valid)
 893 {
 894         struct net2280_dma      *end;
 895         dma_addr_t              tmp;
 896
 897         /* swap new dummy for old, link; fill and maybe activate */
 898         end = ep->dummy;
 899         ep->dummy = req->td;
 900         req->td = end;
 901
 902         tmp = ep->td_dma;
 903         ep->td_dma = req->td_dma;
 904         req->td_dma = tmp;
 905
 906         end->dmadesc = cpu_to_le32 (ep->td_dma);
 907
 908         fill_dma_desc (ep, req, valid);
 909 }
 910
 911 static void
 912 done (struct net2280_ep *ep, struct net2280_request *req, int status)
 913 {
 914         struct net2280          *dev;
 915         unsigned                stopped = ep->stopped;
 916
 917         list_del_init (&req->queue);
 918
 919         if (req->req.status == -EINPROGRESS)
 920                 req->req.status = status;
 921         else
 922                 status = req->req.status;
 923
 924         dev = ep->dev;
 925         if (ep->dma)
 926                 usb_gadget_unmap_request(&dev->gadget, &req->req, ep->is_in);
 927
 928         if (status && status != -ESHUTDOWN)
 929                 VDEBUG (dev, "complete %s req %p stat %d len %u/%u\n",
 930                         ep->ep.name, &req->req, status,
 931                         req->req.actual, req->req.length);
 932
 933         /* don't modify queue heads during completion callback */
 934         ep->stopped = 1;
 935         spin_unlock (&dev->lock);
 936         req->req.complete (&ep->ep, &req->req);
 937         spin_lock (&dev->lock);
 938         ep->stopped = stopped;
 939 }
 940
 941 /*-------------------------------------------------------------------------*/
 942
 943 static int
 944 net2280_queue (struct usb_ep *_ep, struct usb_request *_req, gfp_t gfp_flags)
 945 {
 946         struct net2280_request  *req;
 947         struct net2280_ep       *ep;
 948         struct net2280          *dev;
 949         unsigned long           flags;
 950
 951         /* we always require a cpu-view buffer, so that we can
 952          * always use pio (as fallback or whatever).
 953          */
 954         req = container_of (_req, struct net2280_request, req);
 955         if (!_req || !_req->complete || !_req->buf
 956                         || !list_empty (&req->queue))
 957                 return -EINVAL;
 958         if (_req->length > (~0 & DMA_BYTE_COUNT_MASK))
 959                 return -EDOM;
 960         ep = container_of (_ep, struct net2280_ep, ep);
 961         if (!_ep || (!ep->desc && ep->num != 0))
 962                 return -EINVAL;
 963         dev = ep->dev;
 964         if (!dev->driver || dev->gadget.speed == USB_SPEED_UNKNOWN)
 965                 return -ESHUTDOWN;
 966
 967         /* FIXME implement PIO fallback for ZLPs with DMA */
 968         if (ep->dma && _req->length == 0)
 969                 return -EOPNOTSUPP;
 970
 971         /* set up dma mapping in case the caller didn't */
 972         if (ep->dma) {
 973                 int ret;
 974
 975                 ret = usb_gadget_map_request(&dev->gadget, _req,
 976                                 ep->is_in);
 977                 if (ret)
 978                         return ret;
 979         }
 980
 981 #if 0
 982         VDEBUG (dev, "%s queue req %p, len %d buf %p\n",
 983                         _ep->name, _req, _req->length, _req->buf);
 984 #endif
 985
 986         spin_lock_irqsave (&dev->lock, flags);
 987
 988         _req->status = -EINPROGRESS;
 989         _req->actual = 0;
 990
 991         /* kickstart this i/o queue? */
 992         if (list_empty (&ep->queue) && !ep->stopped) {
 993                 /* DMA request while EP halted */
 994                 if (ep->dma &&
 995                     (readl(&ep->regs->ep_rsp) & (1 << CLEAR_ENDPOINT_HALT)) &&
 996                         (dev->pdev->vendor == 0x10b5)) {
 997                         int valid = 1;
 998                         if (ep->is_in) {
 999                                 int expect;
1000                                 expect = likely(req->req.zero ||
1001                                                 ((req->req.length %
1002                                                   ep->ep.maxpacket) != 0));
1003                                 if (expect != ep->in_fifo_validate)
1004                                         valid = 0;
1005                         }
1006                         queue_dma(ep, req, valid);
1007                 }
1008                 /* use DMA if the endpoint supports it, else pio */
1009                 else if (ep->dma)
1010                         start_dma (ep, req);
1011                 else {
1012                         /* maybe there's no control data, just status ack */
1013                         if (ep->num == 0 && _req->length == 0) {
1014                                 allow_status (ep);
1015                                 done (ep, req, 0);
1016                                 VDEBUG (dev, "%s status ack\n", ep->ep.name);
1017                                 goto done;
1018                         }
1019
1020                         /* PIO ... stuff the fifo, or unblock it.  */
1021                         if (ep->is_in)
1022                                 write_fifo (ep, _req);
1023                         else if (list_empty (&ep->queue)) {
1024                                 u32     s;
1025
1026                                 /* OUT FIFO might have packet(s) buffered */
1027                                 s = readl (&ep->regs->ep_stat);
1028                                 if ((s & (1 << FIFO_EMPTY)) == 0) {
1029                                         /* note:  _req->short_not_ok is
1030                                          * ignored here since PIO _always_
1031                                          * stops queue advance here, and
1032                                          * _req->status doesn't change for
1033                                          * short reads (only _req->actual)
1034                                          */
1035                                         if (read_fifo (ep, req)) {
1036                                                 done (ep, req, 0);
1037                                                 if (ep->num == 0)
1038                                                         allow_status (ep);
1039                                                 /* don't queue it */
1040                                                 req = NULL;
1041                                         } else
1042                                                 s = readl (&ep->regs->ep_stat);
1043                                 }
1044
1045                                 /* don't NAK, let the fifo fill */
1046                                 if (req && (s & (1 << NAK_OUT_PACKETS)))
1047                                         writel ((1 << CLEAR_NAK_OUT_PACKETS),
1048                                                         &ep->regs->ep_rsp);
1049                         }
1050                 }
1051
1052         } else if (ep->dma) {
1053                 int     valid = 1;
1054
1055                 if (ep->is_in) {
1056                         int     expect;
1057
1058                         /* preventing magic zlps is per-engine state, not
1059                          * per-transfer; irq logic must recover hiccups.
1060                          */
1061                         expect = likely (req->req.zero
1062                                 || (req->req.length % ep->ep.maxpacket) != 0);
1063                         if (expect != ep->in_fifo_validate)
1064                                 valid = 0;
1065                 }
1066                 queue_dma (ep, req, valid);
1067
1068         } /* else the irq handler advances the queue. */
1069
1070         ep->responded = 1;
1071         if (req)
1072                 list_add_tail (&req->queue, &ep->queue);
1073 done:
1074         spin_unlock_irqrestore (&dev->lock, flags);
1075
1076         /* pci writes may still be posted */
1077         return 0;
1078 }
1079
1080 static inline void
1081 dma_done (
1082         struct net2280_ep *ep,
1083         struct net2280_request *req,
1084         u32 dmacount,
1085         int status
1086 )
1087 {
1088         req->req.actual = req->req.length - (DMA_BYTE_COUNT_MASK & dmacount);
1089         done (ep, req, status);
1090 }
1091
1092 static void restart_dma (struct net2280_ep *ep);
1093
1094 static void scan_dma_completions (struct net2280_ep *ep)
1095 {
1096         /* only look at descriptors that were "naturally" retired,
1097          * so fifo and list head state won't matter
1098          */
1099         while (!list_empty (&ep->queue)) {
1100                 struct net2280_request  *req;
1101                 u32                     tmp;
1102
1103                 req = list_entry (ep->queue.next,
1104                                 struct net2280_request, queue);
1105                 if (!req->valid)
1106                         break;
1107                 rmb ();
1108                 tmp = le32_to_cpup (&req->td->dmacount);
1109                 if ((tmp & (1 << VALID_BIT)) != 0)
1110                         break;
1111
1112                 /* SHORT_PACKET_TRANSFERRED_INTERRUPT handles "usb-short"
1113                  * cases where DMA must be aborted; this code handles
1114                  * all non-abort DMA completions.
1115                  */
1116                 if (unlikely (req->td->dmadesc == 0)) {
1117                         /* paranoia */
1118                         tmp = readl (&ep->dma->dmacount);
1119                         if (tmp & DMA_BYTE_COUNT_MASK)
1120                                 break;
1121                         /* single transfer mode */
1122                         dma_done (ep, req, tmp, 0);
1123                         break;
1124                 } else if (!ep->is_in
1125                                 && (req->req.length % ep->ep.maxpacket) != 0) {
1126                         tmp = readl (&ep->regs->ep_stat);
1127                         if (ep->dev->pdev->vendor == 0x10b5)
1128                                 return dma_done(ep, req, tmp, 0);
1129
1130                         /* AVOID TROUBLE HERE by not issuing short reads from
1131                          * your gadget driver.  That helps avoids errata 0121,
1132                          * 0122, and 0124; not all cases trigger the warning.
1133                          */
1134                         if ((tmp & (1 << NAK_OUT_PACKETS)) == 0) {
1135                                 WARNING (ep->dev, "%s lost packet sync!\n",
1136                                                 ep->ep.name);
1137                                 req->req.status = -EOVERFLOW;
1138                         } else if ((tmp = readl (&ep->regs->ep_avail)) != 0) {
1139                                 /* fifo gets flushed later */
1140                                 ep->out_overflow = 1;
1141                                 DEBUG (ep->dev, "%s dma, discard %d len %d\n",
1142                                                 ep->ep.name, tmp,
1143                                                 req->req.length);
1144                                 req->req.status = -EOVERFLOW;
1145                         }
1146                 }
1147                 dma_done (ep, req, tmp, 0);
1148         }
1149 }
1150
1151 static void restart_dma (struct net2280_ep *ep)
1152 {
1153         struct net2280_request  *req;
1154         u32                     dmactl = dmactl_default;
1155
1156         if (ep->stopped)
1157                 return;
1158         req = list_entry (ep->queue.next, struct net2280_request, queue);
1159
1160         if (!use_dma_chaining) {
1161                 start_dma (ep, req);
1162                 return;
1163         }
1164
1165         /* the 2280 will be processing the queue unless queue hiccups after
1166          * the previous transfer:
1167          *  IN:   wanted automagic zlp, head doesn't (or vice versa)
1168          *        DMA_FIFO_VALIDATE doesn't init from dma descriptors.
1169          *  OUT:  was "usb-short", we must restart.
1170          */
1171         if (ep->is_in && !req->valid) {
1172                 struct net2280_request  *entry, *prev = NULL;
1173                 int                     reqmode, done = 0;
1174
1175                 DEBUG (ep->dev, "%s dma hiccup td %p\n", ep->ep.name, req->td);
1176                 ep->in_fifo_validate = likely (req->req.zero
1177                         || (req->req.length % ep->ep.maxpacket) != 0);
1178                 if (ep->in_fifo_validate)
1179                         dmactl |= (1 << DMA_FIFO_VALIDATE);
1180                 list_for_each_entry (entry, &ep->queue, queue) {
1181                         __le32          dmacount;
1182
1183                         if (entry == req)
1184                                 continue;
1185                         dmacount = entry->td->dmacount;
1186                         if (!done) {
1187                                 reqmode = likely (entry->req.zero
1188                                         || (entry->req.length
1189                                                 % ep->ep.maxpacket) != 0);
1190                                 if (reqmode == ep->in_fifo_validate) {
1191                                         entry->valid = 1;
1192                                         dmacount |= valid_bit;
1193                                         entry->td->dmacount = dmacount;
1194                                         prev = entry;
1195                                         continue;
1196                                 } else {
1197                                         /* force a hiccup */
1198                                         prev->td->dmacount |= dma_done_ie;
1199                                         done = 1;
1200                                 }
1201                         }
1202
1203                         /* walk the rest of the queue so unlinks behave */
1204                         entry->valid = 0;
1205                         dmacount &= ~valid_bit;
1206                         entry->td->dmacount = dmacount;
1207                         prev = entry;
1208                 }
1209         }
1210
1211         writel (0, &ep->dma->dmactl);
1212         start_queue (ep, dmactl, req->td_dma);
1213 }
1214
1215 static void abort_dma_228x(struct net2280_ep *ep)
1216 {
1217         /* abort the current transfer */
1218         if (likely (!list_empty (&ep->queue))) {
1219                 /* FIXME work around errata 0121, 0122, 0124 */
1220                 writel ((1 << DMA_ABORT), &ep->dma->dmastat);
1221                 spin_stop_dma (ep->dma);
1222         } else
1223                 stop_dma (ep->dma);
1224         scan_dma_completions (ep);
1225 }
1226
1227 static void abort_dma_338x(struct net2280_ep *ep)
1228 {
1229         writel((1 << DMA_ABORT), &ep->dma->dmastat);
1230         spin_stop_dma(ep->dma);
1231 }
1232
1233 static void abort_dma(struct net2280_ep *ep)
1234 {
1235         if (ep->dev->pdev->vendor == 0x17cc)
1236                 return abort_dma_228x(ep);
1237         return abort_dma_338x(ep);
1238 }
1239
1240 /* dequeue ALL requests */
1241 static void nuke (struct net2280_ep *ep)
1242 {
1243         struct net2280_request  *req;
1244
1245         /* called with spinlock held */
1246         ep->stopped = 1;
1247         if (ep->dma)
1248                 abort_dma (ep);
1249         while (!list_empty (&ep->queue)) {
1250                 req = list_entry (ep->queue.next,
1251                                 struct net2280_request,
1252                                 queue);
1253                 done (ep, req, -ESHUTDOWN);
1254         }
1255 }
1256
1257 /* dequeue JUST ONE request */
1258 static int net2280_dequeue (struct usb_ep *_ep, struct usb_request *_req)
1259 {
1260         struct net2280_ep       *ep;
1261         struct net2280_request  *req;
1262         unsigned long           flags;
1263         u32                     dmactl;
1264         int                     stopped;
1265
1266         ep = container_of (_ep, struct net2280_ep, ep);
1267         if (!_ep || (!ep->desc && ep->num != 0) || !_req)
1268                 return -EINVAL;
1269
1270         spin_lock_irqsave (&ep->dev->lock, flags);
1271         stopped = ep->stopped;
1272
1273         /* quiesce dma while we patch the queue */
1274         dmactl = 0;
1275         ep->stopped = 1;
1276         if (ep->dma) {
1277                 dmactl = readl (&ep->dma->dmactl);
1278                 /* WARNING erratum 0127 may kick in ... */
1279                 stop_dma (ep->dma);
1280                 scan_dma_completions (ep);
1281         }
1282
1283         /* make sure it's still queued on this endpoint */
1284         list_for_each_entry (req, &ep->queue, queue) {
1285                 if (&req->req == _req)
1286                         break;
1287         }
1288         if (&req->req != _req) {
1289                 spin_unlock_irqrestore (&ep->dev->lock, flags);
1290                 return -EINVAL;
1291         }
1292
1293         /* queue head may be partially complete. */
1294         if (ep->queue.next == &req->queue) {
1295                 if (ep->dma) {
1296                         DEBUG (ep->dev, "unlink (%s) dma\n", _ep->name);
1297                         _req->status = -ECONNRESET;
1298                         abort_dma (ep);
1299                         if (likely (ep->queue.next == &req->queue)) {
1300                                 // NOTE: misreports single-transfer mode
1301                                 req->td->dmacount = 0;  /* invalidate */
1302                                 dma_done (ep, req,
1303                                         readl (&ep->dma->dmacount),
1304                                         -ECONNRESET);
1305                         }
1306                 } else {
1307                         DEBUG (ep->dev, "unlink (%s) pio\n", _ep->name);
1308                         done (ep, req, -ECONNRESET);
1309                 }
1310                 req = NULL;
1311
1312         /* patch up hardware chaining data */
1313         } else if (ep->dma && use_dma_chaining) {
1314                 if (req->queue.prev == ep->queue.next) {
1315                         writel (le32_to_cpu (req->td->dmadesc),
1316                                 &ep->dma->dmadesc);
1317                         if (req->td->dmacount & dma_done_ie)
1318                                 writel (readl (&ep->dma->dmacount)
1319                                                 | le32_to_cpu(dma_done_ie),
1320                                         &ep->dma->dmacount);
1321                 } else {
1322                         struct net2280_request  *prev;
1323
1324                         prev = list_entry (req->queue.prev,
1325                                 struct net2280_request, queue);
1326                         prev->td->dmadesc = req->td->dmadesc;
1327                         if (req->td->dmacount & dma_done_ie)
1328                                 prev->td->dmacount |= dma_done_ie;
1329                 }
1330         }
1331
1332         if (req)
1333                 done (ep, req, -ECONNRESET);
1334         ep->stopped = stopped;
1335
1336         if (ep->dma) {
1337                 /* turn off dma on inactive queues */
1338                 if (list_empty (&ep->queue))
1339                         stop_dma (ep->dma);
1340                 else if (!ep->stopped) {
1341                         /* resume current request, or start new one */
1342                         if (req)
1343                                 writel (dmactl, &ep->dma->dmactl);
1344                         else
1345                                 start_dma (ep, list_entry (ep->queue.next,
1346                                         struct net2280_request, queue));
1347                 }
1348         }
1349
1350         spin_unlock_irqrestore (&ep->dev->lock, flags);
1351         return 0;
1352 }
1353
1354 /*-------------------------------------------------------------------------*/
1355
1356 static int net2280_fifo_status (struct usb_ep *_ep);
1357
1358 static int
1359 net2280_set_halt_and_wedge(struct usb_ep *_ep, int value, int wedged)
1360 {
1361         struct net2280_ep       *ep;
1362         unsigned long           flags;
1363         int                     retval = 0;
1364
1365         ep = container_of (_ep, struct net2280_ep, ep);
1366         if (!_ep || (!ep->desc && ep->num != 0))
1367                 return -EINVAL;
1368         if (!ep->dev->driver || ep->dev->gadget.speed == USB_SPEED_UNKNOWN)
1369                 return -ESHUTDOWN;
1370         if (ep->desc /* not ep0 */ && (ep->desc->bmAttributes & 0x03)
1371                                                 == USB_ENDPOINT_XFER_ISOC)
1372                 return -EINVAL;
1373
1374         spin_lock_irqsave (&ep->dev->lock, flags);
1375         if (!list_empty (&ep->queue))
1376                 retval = -EAGAIN;
1377         else if (ep->is_in && value && net2280_fifo_status (_ep) != 0)
1378                 retval = -EAGAIN;
1379         else {
1380                 VDEBUG (ep->dev, "%s %s %s\n", _ep->name,
1381                                 value ? "set" : "clear",
1382                                 wedged ? "wedge" : "halt");
1383                 /* set/clear, then synch memory views with the device */
1384                 if (value) {
1385                         if (ep->num == 0)
1386                                 ep->dev->protocol_stall = 1;
1387                         else
1388                                 set_halt (ep);
1389                         if (wedged)
1390                                 ep->wedged = 1;
1391                 } else {
1392                         clear_halt (ep);
1393                         if (ep->dev->pdev->vendor == 0x10b5 &&
1394                                 !list_empty(&ep->queue) && ep->td_dma)
1395                                         restart_dma(ep);
1396                         ep->wedged = 0;
1397                 }
1398                 (void) readl (&ep->regs->ep_rsp);
1399         }
1400         spin_unlock_irqrestore (&ep->dev->lock, flags);
1401
1402         return retval;
1403 }
1404
1405 static int
1406 net2280_set_halt(struct usb_ep *_ep, int value)
1407 {
1408         return net2280_set_halt_and_wedge(_ep, value, 0);
1409 }
1410
1411 static int
1412 net2280_set_wedge(struct usb_ep *_ep)
1413 {
1414         if (!_ep || _ep->name == ep0name)
1415                 return -EINVAL;
1416         return net2280_set_halt_and_wedge(_ep, 1, 1);
1417 }
1418
1419 static int
1420 net2280_fifo_status (struct usb_ep *_ep)
1421 {
1422         struct net2280_ep       *ep;
1423         u32                     avail;
1424
1425         ep = container_of (_ep, struct net2280_ep, ep);
1426         if (!_ep || (!ep->desc && ep->num != 0))
1427                 return -ENODEV;
1428         if (!ep->dev->driver || ep->dev->gadget.speed == USB_SPEED_UNKNOWN)
1429                 return -ESHUTDOWN;
1430
1431         avail = readl (&ep->regs->ep_avail) & ((1 << 12) - 1);
1432         if (avail > ep->fifo_size)
1433                 return -EOVERFLOW;
1434         if (ep->is_in)
1435                 avail = ep->fifo_size - avail;
1436         return avail;
1437 }
1438
1439 static void
1440 net2280_fifo_flush (struct usb_ep *_ep)
1441 {
1442         struct net2280_ep       *ep;
1443
1444         ep = container_of (_ep, struct net2280_ep, ep);
1445         if (!_ep || (!ep->desc && ep->num != 0))
1446                 return;
1447         if (!ep->dev->driver || ep->dev->gadget.speed == USB_SPEED_UNKNOWN)
1448                 return;
1449
1450         writel ((1 << FIFO_FLUSH), &ep->regs->ep_stat);
1451         (void) readl (&ep->regs->ep_rsp);
1452 }
1453
1454 static const struct usb_ep_ops net2280_ep_ops = {
1455         .enable         = net2280_enable,
1456         .disable        = net2280_disable,
1457
1458         .alloc_request  = net2280_alloc_request,
1459         .free_request   = net2280_free_request,
1460
1461         .queue          = net2280_queue,
1462         .dequeue        = net2280_dequeue,
1463
1464         .set_halt       = net2280_set_halt,
1465         .set_wedge      = net2280_set_wedge,
1466         .fifo_status    = net2280_fifo_status,
1467         .fifo_flush     = net2280_fifo_flush,
1468 };
1469
1470 /*-------------------------------------------------------------------------*/
1471
1472 static int net2280_get_frame (struct usb_gadget *_gadget)
1473 {
1474         struct net2280          *dev;
1475         unsigned long           flags;
1476         u16                     retval;
1477
1478         if (!_gadget)
1479                 return -ENODEV;
1480         dev = container_of (_gadget, struct net2280, gadget);
1481         spin_lock_irqsave (&dev->lock, flags);
1482         retval = get_idx_reg (dev->regs, REG_FRAME) & 0x03ff;
1483         spin_unlock_irqrestore (&dev->lock, flags);
1484         return retval;
1485 }
1486
1487 static int net2280_wakeup (struct usb_gadget *_gadget)
1488 {
1489         struct net2280          *dev;
1490         u32                     tmp;
1491         unsigned long           flags;
1492
1493         if (!_gadget)
1494                 return 0;
1495         dev = container_of (_gadget, struct net2280, gadget);
1496
1497         spin_lock_irqsave (&dev->lock, flags);
1498         tmp = readl (&dev->usb->usbctl);
1499         if (tmp & (1 << DEVICE_REMOTE_WAKEUP_ENABLE))
1500                 writel (1 << GENERATE_RESUME, &dev->usb->usbstat);
1501         spin_unlock_irqrestore (&dev->lock, flags);
1502
1503         /* pci writes may still be posted */
1504         return 0;
1505 }
1506
1507 static int net2280_set_selfpowered (struct usb_gadget *_gadget, int value)
1508 {
1509         struct net2280          *dev;
1510         u32                     tmp;
1511         unsigned long           flags;
1512
1513         if (!_gadget)
1514                 return 0;
1515         dev = container_of (_gadget, struct net2280, gadget);
1516
1517         spin_lock_irqsave (&dev->lock, flags);
1518         tmp = readl (&dev->usb->usbctl);
1519         if (value) {
1520                 tmp |= (1 << SELF_POWERED_STATUS);
1521                 dev->selfpowered = 1;
1522         } else {
1523                 tmp &= ~(1 << SELF_POWERED_STATUS);
1524                 dev->selfpowered = 0;
1525         }
1526         writel (tmp, &dev->usb->usbctl);
1527         spin_unlock_irqrestore (&dev->lock, flags);
1528
1529         return 0;
1530 }
1531
1532 static int net2280_pullup(struct usb_gadget *_gadget, int is_on)
1533 {
1534         struct net2280  *dev;
1535         u32             tmp;
1536         unsigned long   flags;
1537
1538         if (!_gadget)
1539                 return -ENODEV;
1540         dev = container_of (_gadget, struct net2280, gadget);
1541
1542         spin_lock_irqsave (&dev->lock, flags);
1543         tmp = readl (&dev->usb->usbctl);
1544         dev->softconnect = (is_on != 0);
1545         if (is_on)
1546                 tmp |= (1 << USB_DETECT_ENABLE);
1547         else
1548                 tmp &= ~(1 << USB_DETECT_ENABLE);
1549         writel (tmp, &dev->usb->usbctl);
1550         spin_unlock_irqrestore (&dev->lock, flags);
1551
1552         return 0;
1553 }
1554
1555 static int net2280_start(struct usb_gadget *_gadget,
1556                 struct usb_gadget_driver *driver);
1557 static int net2280_stop(struct usb_gadget *_gadget,
1558                 struct usb_gadget_driver *driver);
1559
1560 static const struct usb_gadget_ops net2280_ops = {
1561         .get_frame      = net2280_get_frame,
1562         .wakeup         = net2280_wakeup,
1563         .set_selfpowered = net2280_set_selfpowered,
1564         .pullup         = net2280_pullup,
1565         .udc_start      = net2280_start,
1566         .udc_stop       = net2280_stop,
1567 };
1568
1569 /*-------------------------------------------------------------------------*/
1570
1571 #ifdef  CONFIG_USB_GADGET_DEBUG_FILES
1572
1573 /* FIXME move these into procfs, and use seq_file.
1574  * Sysfs _still_ doesn't behave for arbitrarily sized files,
1575  * and also doesn't help products using this with 2.4 kernels.
1576  */
1577
1578 /* "function" sysfs attribute */
1579 static ssize_t function_show(struct device *_dev, struct device_attribute *attr,
1580                              char *buf)
1581 {
1582         struct net2280  *dev = dev_get_drvdata (_dev);
1583
1584         if (!dev->driver
1585                         || !dev->driver->function
1586                         || strlen (dev->driver->function) > PAGE_SIZE)
1587                 return 0;
1588         return scnprintf (buf, PAGE_SIZE, "%s\n", dev->driver->function);
1589 }
1590 static DEVICE_ATTR_RO(function);
1591
1592 static ssize_t registers_show(struct device *_dev,
1593                               struct device_attribute *attr, char *buf)
1594 {
1595         struct net2280          *dev;
1596         char                    *next;
1597         unsigned                size, t;
1598         unsigned long           flags;
1599         int                     i;
1600         u32                     t1, t2;
1601         const char              *s;
1602
1603         dev = dev_get_drvdata (_dev);
1604         next = buf;
1605         size = PAGE_SIZE;
1606         spin_lock_irqsave (&dev->lock, flags);
1607
1608         if (dev->driver)
1609                 s = dev->driver->driver.name;
1610         else
1611                 s = "(none)";
1612
1613         /* Main Control Registers */
1614         t = scnprintf (next, size, "%s version " DRIVER_VERSION
1615                         ", chiprev %04x, dma %s\n\n"
1616                         "devinit %03x fifoctl %08x gadget '%s'\n"
1617                         "pci irqenb0 %02x irqenb1 %08x "
1618                         "irqstat0 %04x irqstat1 %08x\n",
1619                         driver_name, dev->chiprev,
1620                         use_dma
1621                                 ? (use_dma_chaining ? "chaining" : "enabled")
1622                                 : "disabled",
1623                         readl (&dev->regs->devinit),
1624                         readl (&dev->regs->fifoctl),
1625                         s,
1626                         readl (&dev->regs->pciirqenb0),
1627                         readl (&dev->regs->pciirqenb1),
1628                         readl (&dev->regs->irqstat0),
1629                         readl (&dev->regs->irqstat1));
1630         size -= t;
1631         next += t;
1632
1633         /* USB Control Registers */
1634         t1 = readl (&dev->usb->usbctl);
1635         t2 = readl (&dev->usb->usbstat);
1636         if (t1 & (1 << VBUS_PIN)) {
1637                 if (t2 & (1 << HIGH_SPEED))
1638                         s = "high speed";
1639                 else if (dev->gadget.speed == USB_SPEED_UNKNOWN)
1640                         s = "powered";
1641                 else
1642                         s = "full speed";
1643                 /* full speed bit (6) not working?? */
1644         } else
1645                         s = "not attached";
1646         t = scnprintf (next, size,
1647                         "stdrsp %08x usbctl %08x usbstat %08x "
1648                                 "addr 0x%02x (%s)\n",
1649                         readl (&dev->usb->stdrsp), t1, t2,
1650                         readl (&dev->usb->ouraddr), s);
1651         size -= t;
1652         next += t;
1653
1654         /* PCI Master Control Registers */
1655
1656         /* DMA Control Registers */
1657
1658         /* Configurable EP Control Registers */
1659         for (i = 0; i < dev->n_ep; i++) {
1660                 struct net2280_ep       *ep;
1661
1662                 ep = &dev->ep [i];
1663                 if (i && !ep->desc)
1664                         continue;
1665
1666                 t1 = readl(&ep->cfg->ep_cfg);
1667                 t2 = readl (&ep->regs->ep_rsp) & 0xff;
1668                 t = scnprintf (next, size,
1669                                 "\n%s\tcfg %05x rsp (%02x) %s%s%s%s%s%s%s%s"
1670                                         "irqenb %02x\n",
1671                                 ep->ep.name, t1, t2,
1672                                 (t2 & (1 << CLEAR_NAK_OUT_PACKETS))
1673                                         ? "NAK " : "",
1674                                 (t2 & (1 << CLEAR_EP_HIDE_STATUS_PHASE))
1675                                         ? "hide " : "",
1676                                 (t2 & (1 << CLEAR_EP_FORCE_CRC_ERROR))
1677                                         ? "CRC " : "",
1678                                 (t2 & (1 << CLEAR_INTERRUPT_MODE))
1679                                         ? "interrupt " : "",
1680                                 (t2 & (1<<CLEAR_CONTROL_STATUS_PHASE_HANDSHAKE))
1681                                         ? "status " : "",
1682                                 (t2 & (1 << CLEAR_NAK_OUT_PACKETS_MODE))
1683                                         ? "NAKmode " : "",
1684                                 (t2 & (1 << CLEAR_ENDPOINT_TOGGLE))
1685                                         ? "DATA1 " : "DATA0 ",
1686                                 (t2 & (1 << CLEAR_ENDPOINT_HALT))
1687                                         ? "HALT " : "",
1688                                 readl (&ep->regs->ep_irqenb));
1689                 size -= t;
1690                 next += t;
1691
1692                 t = scnprintf (next, size,
1693                                 "\tstat %08x avail %04x "
1694                                 "(ep%d%s-%s)%s\n",
1695                                 readl (&ep->regs->ep_stat),
1696                                 readl (&ep->regs->ep_avail),
1697                                 t1 & 0x0f, DIR_STRING (t1),
1698                                 type_string (t1 >> 8),
1699                                 ep->stopped ? "*" : "");
1700                 size -= t;
1701                 next += t;
1702
1703                 if (!ep->dma)
1704                         continue;
1705
1706                 t = scnprintf (next, size,
1707                                 "  dma\tctl %08x stat %08x count %08x\n"
1708                                 "\taddr %08x desc %08x\n",
1709                                 readl (&ep->dma->dmactl),
1710                                 readl (&ep->dma->dmastat),
1711                                 readl (&ep->dma->dmacount),
1712                                 readl (&ep->dma->dmaaddr),
1713                                 readl (&ep->dma->dmadesc));
1714                 size -= t;
1715                 next += t;
1716
1717         }
1718
1719         /* Indexed Registers */
1720                 // none yet
1721
1722         /* Statistics */
1723         t = scnprintf (next, size, "\nirqs:  ");
1724         size -= t;
1725         next += t;
1726         for (i = 0; i < dev->n_ep; i++) {
1727                 struct net2280_ep       *ep;
1728
1729                 ep = &dev->ep [i];
1730                 if (i && !ep->irqs)
1731                         continue;
1732                 t = scnprintf (next, size, " %s/%lu", ep->ep.name, ep->irqs);
1733                 size -= t;
1734                 next += t;
1735
1736         }
1737         t = scnprintf (next, size, "\n");
1738         size -= t;
1739         next += t;
1740
1741         spin_unlock_irqrestore (&dev->lock, flags);
1742
1743         return PAGE_SIZE - size;
1744 }
1745 static DEVICE_ATTR_RO(registers);
1746
1747 static ssize_t queues_show(struct device *_dev, struct device_attribute *attr,
1748                            char *buf)
1749 {
1750         struct net2280          *dev;
1751         char                    *next;
1752         unsigned                size;
1753         unsigned long           flags;
1754         int                     i;
1755
1756         dev = dev_get_drvdata (_dev);
1757         next = buf;
1758         size = PAGE_SIZE;
1759         spin_lock_irqsave (&dev->lock, flags);
1760
1761         for (i = 0; i < dev->n_ep; i++) {
1762                 struct net2280_ep               *ep = &dev->ep [i];
1763                 struct net2280_request          *req;
1764                 int                             t;
1765
1766                 if (i != 0) {
1767                         const struct usb_endpoint_descriptor    *d;
1768
1769                         d = ep->desc;
1770                         if (!d)
1771                                 continue;
1772                         t = d->bEndpointAddress;
1773                         t = scnprintf (next, size,
1774                                 "\n%s (ep%d%s-%s) max %04x %s fifo %d\n",
1775                                 ep->ep.name, t & USB_ENDPOINT_NUMBER_MASK,
1776                                 (t & USB_DIR_IN) ? "in" : "out",
1777                                 ({ char *val;
1778                                  switch (d->bmAttributes & 0x03) {
1779                                  case USB_ENDPOINT_XFER_BULK:
1780                                         val = "bulk"; break;
1781                                  case USB_ENDPOINT_XFER_INT:
1782                                         val = "intr"; break;
1783                                  default:
1784                                         val = "iso"; break;
1785                                  } val; }),
1786                                 usb_endpoint_maxp (d) & 0x1fff,
1787                                 ep->dma ? "dma" : "pio", ep->fifo_size
1788                                 );
1789                 } else /* ep0 should only have one transfer queued */
1790                         t = scnprintf (next, size, "ep0 max 64 pio %s\n",
1791                                         ep->is_in ? "in" : "out");
1792                 if (t <= 0 || t > size)
1793                         goto done;
1794                 size -= t;
1795                 next += t;
1796
1797                 if (list_empty (&ep->queue)) {
1798                         t = scnprintf (next, size, "\t(nothing queued)\n");
1799                         if (t <= 0 || t > size)
1800                                 goto done;
1801                         size -= t;
1802                         next += t;
1803                         continue;
1804                 }
1805                 list_for_each_entry (req, &ep->queue, queue) {
1806                         if (ep->dma && req->td_dma == readl (&ep->dma->dmadesc))
1807                                 t = scnprintf (next, size,
1808                                         "\treq %p len %d/%d "
1809                                         "buf %p (dmacount %08x)\n",
1810                                         &req->req, req->req.actual,
1811                                         req->req.length, req->req.buf,
1812                                         readl (&ep->dma->dmacount));
1813                         else
1814                                 t = scnprintf (next, size,
1815                                         "\treq %p len %d/%d buf %p\n",
1816                                         &req->req, req->req.actual,
1817                                         req->req.length, req->req.buf);
1818                         if (t <= 0 || t > size)
1819                                 goto done;
1820                         size -= t;
1821                         next += t;
1822
1823                         if (ep->dma) {
1824                                 struct net2280_dma      *td;
1825
1826                                 td = req->td;
1827                                 t = scnprintf (next, size, "\t    td %08x "
1828                                         " count %08x buf %08x desc %08x\n",
1829                                         (u32) req->td_dma,
1830                                         le32_to_cpu (td->dmacount),
1831                                         le32_to_cpu (td->dmaaddr),
1832                                         le32_to_cpu (td->dmadesc));
1833                                 if (t <= 0 || t > size)
1834                                         goto done;
1835                                 size -= t;
1836                                 next += t;
1837                         }
1838                 }
1839         }
1840
1841 done:
1842         spin_unlock_irqrestore (&dev->lock, flags);
1843         return PAGE_SIZE - size;
1844 }
1845 static DEVICE_ATTR_RO(queues);
1846
1847
1848 #else
1849
1850 #define device_create_file(a,b) (0)
1851 #define device_remove_file(a,b) do { } while (0)
1852
1853 #endif
1854
1855 /*-------------------------------------------------------------------------*/
1856
1857 /* another driver-specific mode might be a request type doing dma
1858  * to/from another device fifo instead of to/from memory.
1859  */
1860
1861 static void set_fifo_mode (struct net2280 *dev, int mode)
1862 {
1863         /* keeping high bits preserves BAR2 */
1864         writel ((0xffff << PCI_BASE2_RANGE) | mode, &dev->regs->fifoctl);
1865
1866         /* always ep-{a,b,e,f} ... maybe not ep-c or ep-d */
1867         INIT_LIST_HEAD (&dev->gadget.ep_list);
1868         list_add_tail (&dev->ep [1].ep.ep_list, &dev->gadget.ep_list);
1869         list_add_tail (&dev->ep [2].ep.ep_list, &dev->gadget.ep_list);
1870         switch (mode) {
1871         case 0:
1872                 list_add_tail (&dev->ep [3].ep.ep_list, &dev->gadget.ep_list);
1873                 list_add_tail (&dev->ep [4].ep.ep_list, &dev->gadget.ep_list);
1874                 dev->ep [1].fifo_size = dev->ep [2].fifo_size = 1024;
1875                 break;
1876         case 1:
1877                 dev->ep [1].fifo_size = dev->ep [2].fifo_size = 2048;
1878                 break;
1879         case 2:
1880                 list_add_tail (&dev->ep [3].ep.ep_list, &dev->gadget.ep_list);
1881                 dev->ep [1].fifo_size = 2048;
1882                 dev->ep [2].fifo_size = 1024;
1883                 break;
1884         }
1885         /* fifo sizes for ep0, ep-c, ep-d, ep-e, and ep-f never change */
1886         list_add_tail (&dev->ep [5].ep.ep_list, &dev->gadget.ep_list);
1887         list_add_tail (&dev->ep [6].ep.ep_list, &dev->gadget.ep_list);
1888 }
1889
1890 static void defect7374_disable_data_eps(struct net2280 *dev)
1891 {
1892         /*
1893          * For Defect 7374, disable data EPs (and more):
1894          *  - This phase undoes the earlier phase of the Defect 7374 workaround,
1895          *    returing ep regs back to normal.
1896          */
1897         struct net2280_ep *ep;
1898         int i;
1899         unsigned char ep_sel;
1900         u32 tmp_reg;
1901
1902         for (i = 1; i < 5; i++) {
1903                 ep = &dev->ep[i];
1904                 writel(0, &ep->cfg->ep_cfg);
1905         }
1906
1907         /* CSROUT, CSRIN, PCIOUT, PCIIN, STATIN, RCIN */
1908         for (i = 0; i < 6; i++)
1909                 writel(0, &dev->dep[i].dep_cfg);
1910
1911         for (ep_sel = 0; ep_sel <= 21; ep_sel++) {
1912                 /* Select an endpoint for subsequent operations: */
1913                 tmp_reg = readl(&dev->plregs->pl_ep_ctrl);
1914                 writel(((tmp_reg & ~0x1f) | ep_sel), &dev->plregs->pl_ep_ctrl);
1915
1916                 if (ep_sel < 2 || (ep_sel > 9 && ep_sel < 14) ||
1917                                         ep_sel == 18 || ep_sel == 20)
1918                         continue;
1919
1920                 /* Change settings on some selected endpoints */
1921                 tmp_reg = readl(&dev->plregs->pl_ep_cfg_4);
1922                 tmp_reg &= ~(1 << NON_CTRL_IN_TOLERATE_BAD_DIR);
1923                 writel(tmp_reg, &dev->plregs->pl_ep_cfg_4);
1924                 tmp_reg = readl(&dev->plregs->pl_ep_ctrl);
1925                 tmp_reg |= (1 << EP_INITIALIZED);
1926                 writel(tmp_reg, &dev->plregs->pl_ep_ctrl);
1927         }
1928 }
1929
1930 static void defect7374_enable_data_eps_zero(struct net2280 *dev)
1931 {
1932         u32 tmp = 0, tmp_reg;
1933         u32 fsmvalue, scratch;
1934         int i;
1935         unsigned char ep_sel;
1936
1937         scratch = get_idx_reg(dev->regs, SCRATCH);
1938         fsmvalue = scratch & (0xf << DEFECT7374_FSM_FIELD);
1939         scratch &= ~(0xf << DEFECT7374_FSM_FIELD);
1940
1941         /*See if firmware needs to set up for workaround*/
1942         if (fsmvalue != DEFECT7374_FSM_SS_CONTROL_READ) {
1943                 WARNING(dev, "Operate Defect 7374 workaround soft this time");
1944                 WARNING(dev, "It will operate on cold-reboot and SS connect");
1945
1946                 /*GPEPs:*/
1947                 tmp = ((0 << ENDPOINT_NUMBER) | (1 << ENDPOINT_DIRECTION) |
1948                        (2 << OUT_ENDPOINT_TYPE) | (2 << IN_ENDPOINT_TYPE) |
1949                        ((dev->enhanced_mode) ?
1950                         1 << OUT_ENDPOINT_ENABLE : 1 << ENDPOINT_ENABLE) |
1951                        (1 << IN_ENDPOINT_ENABLE));
1952
1953                 for (i = 1; i < 5; i++)
1954                         writel(tmp, &dev->ep[i].cfg->ep_cfg);
1955
1956                 /* CSRIN, PCIIN, STATIN, RCIN*/
1957                 tmp = ((0 << ENDPOINT_NUMBER) | (1 << ENDPOINT_ENABLE));
1958                 writel(tmp, &dev->dep[1].dep_cfg);
1959                 writel(tmp, &dev->dep[3].dep_cfg);
1960                 writel(tmp, &dev->dep[4].dep_cfg);
1961                 writel(tmp, &dev->dep[5].dep_cfg);
1962
1963                 /*Implemented for development and debug.
1964                  * Can be refined/tuned later.*/
1965                 for (ep_sel = 0; ep_sel <= 21; ep_sel++) {
1966                         /* Select an endpoint for subsequent operations: */
1967                         tmp_reg = readl(&dev->plregs->pl_ep_ctrl);
1968                         writel(((tmp_reg & ~0x1f) | ep_sel),
1969                                &dev->plregs->pl_ep_ctrl);
1970
1971                         if (ep_sel == 1) {
1972                                 tmp =
1973                                     (readl(&dev->plregs->pl_ep_ctrl) |
1974                                      (1 << CLEAR_ACK_ERROR_CODE) | 0);
1975                                 writel(tmp, &dev->plregs->pl_ep_ctrl);
1976                                 continue;
1977                         }
1978
1979                         if (ep_sel == 0 || (ep_sel > 9 && ep_sel < 14) ||
1980                                         ep_sel == 18  || ep_sel == 20)
1981                                 continue;
1982
1983                         tmp = (readl(&dev->plregs->pl_ep_cfg_4) |
1984                                  (1 << NON_CTRL_IN_TOLERATE_BAD_DIR) | 0);
1985                         writel(tmp, &dev->plregs->pl_ep_cfg_4);
1986
1987                         tmp = readl(&dev->plregs->pl_ep_ctrl) &
1988                                 ~(1 << EP_INITIALIZED);
1989                         writel(tmp, &dev->plregs->pl_ep_ctrl);
1990
1991                 }
1992
1993                 /* Set FSM to focus on the first Control Read:
1994                  * - Tip: Connection speed is known upon the first
1995                  * setup request.*/
1996                 scratch |= DEFECT7374_FSM_WAITING_FOR_CONTROL_READ;
1997                 set_idx_reg(dev->regs, SCRATCH, scratch);
1998
1999         } else{
2000                 WARNING(dev, "Defect 7374 workaround soft will NOT operate");
2001                 WARNING(dev, "It will operate on cold-reboot and SS connect");
2002         }
2003 }
2004
2005 /* keeping it simple:
2006  * - one bus driver, initted first;
2007  * - one function driver, initted second
2008  *
2009  * most of the work to support multiple net2280 controllers would
2010  * be to associate this gadget driver (yes?) with all of them, or
2011  * perhaps to bind specific drivers to specific devices.
2012  */
2013
2014 static void usb_reset_228x(struct net2280 *dev)
2015 {
2016         u32     tmp;
2017
2018         dev->gadget.speed = USB_SPEED_UNKNOWN;
2019         (void) readl (&dev->usb->usbctl);
2020
2021         net2280_led_init (dev);
2022
2023         /* disable automatic responses, and irqs */
2024         writel (0, &dev->usb->stdrsp);
2025         writel (0, &dev->regs->pciirqenb0);
2026         writel (0, &dev->regs->pciirqenb1);
2027
2028         /* clear old dma and irq state */
2029         for (tmp = 0; tmp < 4; tmp++) {
2030                 struct net2280_ep       *ep = &dev->ep[tmp + 1];
2031                 if (ep->dma)
2032                         abort_dma(ep);
2033         }
2034
2035         writel (~0, &dev->regs->irqstat0),
2036         writel (~(1 << SUSPEND_REQUEST_INTERRUPT), &dev->regs->irqstat1),
2037
2038         /* reset, and enable pci */
2039         tmp = readl (&dev->regs->devinit)
2040                 | (1 << PCI_ENABLE)
2041                 | (1 << FIFO_SOFT_RESET)
2042                 | (1 << USB_SOFT_RESET)
2043                 | (1 << M8051_RESET);
2044         writel (tmp, &dev->regs->devinit);
2045
2046         /* standard fifo and endpoint allocations */
2047         set_fifo_mode (dev, (fifo_mode <= 2) ? fifo_mode : 0);
2048 }
2049
2050 static void usb_reset_338x(struct net2280 *dev)
2051 {
2052         u32 tmp;
2053         u32 fsmvalue;
2054
2055         dev->gadget.speed = USB_SPEED_UNKNOWN;
2056         (void)readl(&dev->usb->usbctl);
2057
2058         net2280_led_init(dev);
2059
2060         fsmvalue = get_idx_reg(dev->regs, SCRATCH) &
2061                         (0xf << DEFECT7374_FSM_FIELD);
2062
2063         /* See if firmware needs to set up for workaround: */
2064         if (fsmvalue != DEFECT7374_FSM_SS_CONTROL_READ) {
2065                 INFO(dev, "%s: Defect 7374 FsmValue 0x%08X\n", __func__,
2066                      fsmvalue);
2067         } else {
2068                 /* disable automatic responses, and irqs */
2069                 writel(0, &dev->usb->stdrsp);
2070                 writel(0, &dev->regs->pciirqenb0);
2071                 writel(0, &dev->regs->pciirqenb1);
2072         }
2073
2074         /* clear old dma and irq state */
2075         for (tmp = 0; tmp < 4; tmp++) {
2076                 struct net2280_ep *ep = &dev->ep[tmp + 1];
2077
2078                 if (ep->dma)
2079                         abort_dma(ep);
2080         }
2081
2082         writel(~0, &dev->regs->irqstat0), writel(~0, &dev->regs->irqstat1);
2083
2084         if (fsmvalue == DEFECT7374_FSM_SS_CONTROL_READ) {
2085                 /* reset, and enable pci */
2086                 tmp = readl(&dev->regs->devinit) |
2087                     (1 << PCI_ENABLE) |
2088                     (1 << FIFO_SOFT_RESET) |
2089                     (1 << USB_SOFT_RESET) |
2090                     (1 << M8051_RESET);
2091
2092                 writel(tmp, &dev->regs->devinit);
2093         }
2094
2095         /* always ep-{1,2,3,4} ... maybe not ep-3 or ep-4 */
2096         INIT_LIST_HEAD(&dev->gadget.ep_list);
2097
2098         for (tmp = 1; tmp < dev->n_ep; tmp++)
2099                 list_add_tail(&dev->ep[tmp].ep.ep_list, &dev->gadget.ep_list);
2100
2101 }
2102
2103 static void usb_reset(struct net2280 *dev)
2104 {
2105         if (dev->pdev->vendor == 0x17cc)
2106                 return usb_reset_228x(dev);
2107         return usb_reset_338x(dev);
2108 }
2109
2110 static void usb_reinit_228x(struct net2280 *dev)
2111 {
2112         u32     tmp;
2113         int     init_dma;
2114
2115         /* use_dma changes are ignored till next device re-init */
2116         init_dma = use_dma;
2117
2118         /* basic endpoint init */
2119         for (tmp = 0; tmp < 7; tmp++) {
2120                 struct net2280_ep       *ep = &dev->ep [tmp];
2121
2122                 ep->ep.name = ep_name [tmp];
2123                 ep->dev = dev;
2124                 ep->num = tmp;
2125
2126                 if (tmp > 0 && tmp <= 4) {
2127                         ep->fifo_size = 1024;
2128                         if (init_dma)
2129                                 ep->dma = &dev->dma [tmp - 1];
2130                 } else
2131                         ep->fifo_size = 64;
2132                 ep->regs = &dev->epregs [tmp];
2133                 ep->cfg = &dev->epregs[tmp];
2134                 ep_reset_228x(dev->regs, ep);
2135         }
2136         usb_ep_set_maxpacket_limit(&dev->ep [0].ep, 64);
2137         usb_ep_set_maxpacket_limit(&dev->ep [5].ep, 64);
2138         usb_ep_set_maxpacket_limit(&dev->ep [6].ep, 64);
2139
2140         dev->gadget.ep0 = &dev->ep [0].ep;
2141         dev->ep [0].stopped = 0;
2142         INIT_LIST_HEAD (&dev->gadget.ep0->ep_list);
2143
2144         /* we want to prevent lowlevel/insecure access from the USB host,
2145          * but erratum 0119 means this enable bit is ignored
2146          */
2147         for (tmp = 0; tmp < 5; tmp++)
2148                 writel (EP_DONTUSE, &dev->dep [tmp].dep_cfg);
2149 }
2150
2151 static void usb_reinit_338x(struct net2280 *dev)
2152 {
2153         int init_dma;
2154         int i;
2155         u32 tmp, val;
2156         u32 fsmvalue;
2157         static const u32 ne[9] = { 0, 1, 2, 3, 4, 1, 2, 3, 4 };
2158         static const u32 ep_reg_addr[9] = { 0x00, 0xC0, 0x00, 0xC0, 0x00,
2159                                                 0x00, 0xC0, 0x00, 0xC0 };
2160
2161         /* use_dma changes are ignored till next device re-init */
2162         init_dma = use_dma;
2163
2164         /* basic endpoint init */
2165         for (i = 0; i < dev->n_ep; i++) {
2166                 struct net2280_ep *ep = &dev->ep[i];
2167
2168                 ep->ep.name = ep_name[i];
2169                 ep->dev = dev;
2170                 ep->num = i;
2171
2172                 if (i > 0 && i <= 4 && init_dma)
2173                         ep->dma = &dev->dma[i - 1];
2174
2175                 if (dev->enhanced_mode) {
2176                         ep->cfg = &dev->epregs[ne[i]];
2177                         ep->regs = (struct net2280_ep_regs __iomem *)
2178                                 (((void *)&dev->epregs[ne[i]]) +
2179                                 ep_reg_addr[i]);
2180                         ep->fiforegs = &dev->fiforegs[i];
2181                 } else {
2182                         ep->cfg = &dev->epregs[i];
2183                         ep->regs = &dev->epregs[i];
2184                         ep->fiforegs = &dev->fiforegs[i];
2185                 }
2186
2187                 ep->fifo_size = (i != 0) ? 2048 : 512;
2188
2189                 ep_reset_338x(dev->regs, ep);
2190         }
2191         usb_ep_set_maxpacket_limit(&dev->ep[0].ep, 512);
2192
2193         dev->gadget.ep0 = &dev->ep[0].ep;
2194         dev->ep[0].stopped = 0;
2195
2196         /* Link layer set up */
2197         fsmvalue = get_idx_reg(dev->regs, SCRATCH) &
2198                                 (0xf << DEFECT7374_FSM_FIELD);
2199
2200         /* See if driver needs to set up for workaround: */
2201         if (fsmvalue != DEFECT7374_FSM_SS_CONTROL_READ)
2202                 INFO(dev, "%s: Defect 7374 FsmValue %08x\n",
2203                                                 __func__, fsmvalue);
2204         else {
2205                 tmp = readl(&dev->usb_ext->usbctl2) &
2206                     ~((1 << U1_ENABLE) | (1 << U2_ENABLE) | (1 << LTM_ENABLE));
2207                 writel(tmp, &dev->usb_ext->usbctl2);
2208         }
2209
2210         /* Hardware Defect and Workaround */
2211         val = readl(&dev->ll_lfps_regs->ll_lfps_5);
2212         val &= ~(0xf << TIMER_LFPS_6US);
2213         val |= 0x5 << TIMER_LFPS_6US;
2214         writel(val, &dev->ll_lfps_regs->ll_lfps_5);
2215
2216         val = readl(&dev->ll_lfps_regs->ll_lfps_6);
2217         val &= ~(0xffff << TIMER_LFPS_80US);
2218         val |= 0x0100 << TIMER_LFPS_80US;
2219         writel(val, &dev->ll_lfps_regs->ll_lfps_6);
2220
2221         /*
2222          * AA_AB Errata. Issue 4. Workaround for SuperSpeed USB
2223          * Hot Reset Exit Handshake may Fail in Specific Case using
2224          * Default Register Settings. Workaround for Enumeration test.
2225          */
2226         val = readl(&dev->ll_tsn_regs->ll_tsn_counters_2);
2227         val &= ~(0x1f << HOT_TX_NORESET_TS2);
2228         val |= 0x10 << HOT_TX_NORESET_TS2;
2229         writel(val, &dev->ll_tsn_regs->ll_tsn_counters_2);
2230
2231         val = readl(&dev->ll_tsn_regs->ll_tsn_counters_3);
2232         val &= ~(0x1f << HOT_RX_RESET_TS2);
2233         val |= 0x3 << HOT_RX_RESET_TS2;
2234         writel(val, &dev->ll_tsn_regs->ll_tsn_counters_3);
2235
2236         /*
2237          * Set Recovery Idle to Recover bit:
2238          * - On SS connections, setting Recovery Idle to Recover Fmw improves
2239          *   link robustness with various hosts and hubs.
2240          * - It is safe to set for all connection speeds; all chip revisions.
2241          * - R-M-W to leave other bits undisturbed.
2242          * - Reference PLX TT-7372
2243         */
2244         val = readl(&dev->ll_chicken_reg->ll_tsn_chicken_bit);
2245         val |= (1 << RECOVERY_IDLE_TO_RECOVER_FMW);
2246         writel(val, &dev->ll_chicken_reg->ll_tsn_chicken_bit);
2247
2248         INIT_LIST_HEAD(&dev->gadget.ep0->ep_list);
2249
2250         /* disable dedicated endpoints */
2251         writel(0x0D, &dev->dep[0].dep_cfg);
2252         writel(0x0D, &dev->dep[1].dep_cfg);
2253         writel(0x0E, &dev->dep[2].dep_cfg);
2254         writel(0x0E, &dev->dep[3].dep_cfg);
2255         writel(0x0F, &dev->dep[4].dep_cfg);
2256         writel(0x0C, &dev->dep[5].dep_cfg);
2257 }
2258
2259 static void usb_reinit(struct net2280 *dev)
2260 {
2261         if (dev->pdev->vendor == 0x17cc)
2262                 return usb_reinit_228x(dev);
2263         return usb_reinit_338x(dev);
2264 }
2265
2266 static void ep0_start_228x(struct net2280 *dev)
2267 {
2268         writel (  (1 << CLEAR_EP_HIDE_STATUS_PHASE)
2269                 | (1 << CLEAR_NAK_OUT_PACKETS)
2270                 | (1 << CLEAR_CONTROL_STATUS_PHASE_HANDSHAKE)
2271                 , &dev->epregs [0].ep_rsp);
2272
2273         /*
2274          * hardware optionally handles a bunch of standard requests
2275          * that the API hides from drivers anyway.  have it do so.
2276          * endpoint status/features are handled in software, to
2277          * help pass tests for some dubious behavior.
2278          */
2279         writel (  (1 << SET_TEST_MODE)
2280                 | (1 << SET_ADDRESS)
2281                 | (1 << DEVICE_SET_CLEAR_DEVICE_REMOTE_WAKEUP)
2282                 | (1 << GET_DEVICE_STATUS)
2283                 | (1 << GET_INTERFACE_STATUS)
2284                 , &dev->usb->stdrsp);
2285         writel (  (1 << USB_ROOT_PORT_WAKEUP_ENABLE)
2286                 | (1 << SELF_POWERED_USB_DEVICE)
2287                 | (1 << REMOTE_WAKEUP_SUPPORT)
2288                 | (dev->softconnect << USB_DETECT_ENABLE)
2289                 | (1 << SELF_POWERED_STATUS)
2290                 , &dev->usb->usbctl);
2291
2292         /* enable irqs so we can see ep0 and general operation  */
2293         writel (  (1 << SETUP_PACKET_INTERRUPT_ENABLE)
2294                 | (1 << ENDPOINT_0_INTERRUPT_ENABLE)
2295                 , &dev->regs->pciirqenb0);
2296         writel (  (1 << PCI_INTERRUPT_ENABLE)
2297                 | (1 << PCI_MASTER_ABORT_RECEIVED_INTERRUPT_ENABLE)
2298                 | (1 << PCI_TARGET_ABORT_RECEIVED_INTERRUPT_ENABLE)
2299                 | (1 << PCI_RETRY_ABORT_INTERRUPT_ENABLE)
2300                 | (1 << VBUS_INTERRUPT_ENABLE)
2301                 | (1 << ROOT_PORT_RESET_INTERRUPT_ENABLE)
2302                 | (1 << SUSPEND_REQUEST_CHANGE_INTERRUPT_ENABLE)
2303                 , &dev->regs->pciirqenb1);
2304
2305         /* don't leave any writes posted */
2306         (void) readl (&dev->usb->usbctl);
2307 }
2308
2309 static void ep0_start_338x(struct net2280 *dev)
2310 {
2311         u32 fsmvalue;
2312
2313         fsmvalue = get_idx_reg(dev->regs, SCRATCH) &
2314                         (0xf << DEFECT7374_FSM_FIELD);
2315
2316         if (fsmvalue != DEFECT7374_FSM_SS_CONTROL_READ)
2317                 INFO(dev, "%s: Defect 7374 FsmValue %08x\n", __func__,
2318                      fsmvalue);
2319         else
2320                 writel((1 << CLEAR_NAK_OUT_PACKETS_MODE) |
2321                        (1 << SET_EP_HIDE_STATUS_PHASE),
2322                        &dev->epregs[0].ep_rsp);
2323
2324         /*
2325          * hardware optionally handles a bunch of standard requests
2326          * that the API hides from drivers anyway.  have it do so.
2327          * endpoint status/features are handled in software, to
2328          * help pass tests for some dubious behavior.
2329          */
2330         writel((1 << SET_ISOCHRONOUS_DELAY) |
2331                (1 << SET_SEL) |
2332                (1 << SET_TEST_MODE) |
2333                (1 << SET_ADDRESS) |
2334                (1 << GET_INTERFACE_STATUS) |
2335                (1 << GET_DEVICE_STATUS),
2336                 &dev->usb->stdrsp);
2337         dev->wakeup_enable = 1;
2338         writel((1 << USB_ROOT_PORT_WAKEUP_ENABLE) |
2339                (dev->softconnect << USB_DETECT_ENABLE) |
2340                (1 << DEVICE_REMOTE_WAKEUP_ENABLE),
2341                &dev->usb->usbctl);
2342
2343         /* enable irqs so we can see ep0 and general operation  */
2344         writel((1 << SETUP_PACKET_INTERRUPT_ENABLE) |
2345                (1 << ENDPOINT_0_INTERRUPT_ENABLE)
2346                , &dev->regs->pciirqenb0);
2347         writel((1 << PCI_INTERRUPT_ENABLE) |
2348                (1 << ROOT_PORT_RESET_INTERRUPT_ENABLE) |
2349                (1 << SUSPEND_REQUEST_CHANGE_INTERRUPT_ENABLE) |
2350                (1 << VBUS_INTERRUPT_ENABLE),
2351                &dev->regs->pciirqenb1);
2352
2353         /* don't leave any writes posted */
2354         (void)readl(&dev->usb->usbctl);
2355 }
2356
2357 static void ep0_start(struct net2280 *dev)
2358 {
2359         if (dev->pdev->vendor == 0x17cc)
2360                 return ep0_start_228x(dev);
2361         return ep0_start_338x(dev);
2362 }
2363
2364 /* when a driver is successfully registered, it will receive
2365  * control requests including set_configuration(), which enables
2366  * non-control requests.  then usb traffic follows until a
2367  * disconnect is reported.  then a host may connect again, or
2368  * the driver might get unbound.
2369  */
2370 static int net2280_start(struct usb_gadget *_gadget,
2371                 struct usb_gadget_driver *driver)
2372 {
2373         struct net2280          *dev;
2374         int                     retval;
2375         unsigned                i;
2376
2377         /* insist on high speed support from the driver, since
2378          * (dev->usb->xcvrdiag & FORCE_FULL_SPEED_MODE)
2379          * "must not be used in normal operation"
2380          */
2381         if (!driver || driver->max_speed < USB_SPEED_HIGH
2382                         || !driver->setup)
2383                 return -EINVAL;
2384
2385         dev = container_of (_gadget, struct net2280, gadget);
2386
2387         for (i = 0; i < dev->n_ep; i++)
2388                 dev->ep [i].irqs = 0;
2389
2390         /* hook up the driver ... */
2391         dev->softconnect = 1;
2392         driver->driver.bus = NULL;
2393         dev->driver = driver;
2394
2395         retval = device_create_file (&dev->pdev->dev, &dev_attr_function);
2396         if (retval) goto err_unbind;
2397         retval = device_create_file (&dev->pdev->dev, &dev_attr_queues);
2398         if (retval) goto err_func;
2399
2400         /* Enable force-full-speed testing mode, if desired */
2401         if (full_speed && dev->pdev->vendor == 0x17cc)
2402                 writel(1 << FORCE_FULL_SPEED_MODE, &dev->usb->xcvrdiag);
2403
2404         /* ... then enable host detection and ep0; and we're ready
2405          * for set_configuration as well as eventual disconnect.
2406          */
2407         net2280_led_active (dev, 1);
2408
2409         if (dev->pdev->vendor == 0x10b5)
2410                 defect7374_enable_data_eps_zero(dev);
2411
2412         ep0_start (dev);
2413
2414         DEBUG (dev, "%s ready, usbctl %08x stdrsp %08x\n",
2415                         driver->driver.name,
2416                         readl (&dev->usb->usbctl),
2417                         readl (&dev->usb->stdrsp));
2418
2419         /* pci writes may still be posted */
2420         return 0;
2421
2422 err_func:
2423         device_remove_file (&dev->pdev->dev, &dev_attr_function);
2424 err_unbind:
2425         dev->driver = NULL;
2426         return retval;
2427 }
2428
2429 static void
2430 stop_activity (struct net2280 *dev, struct usb_gadget_driver *driver)
2431 {
2432         int                     i;
2433
2434         /* don't disconnect if it's not connected */
2435         if (dev->gadget.speed == USB_SPEED_UNKNOWN)
2436                 driver = NULL;
2437
2438         /* stop hardware; prevent new request submissions;
2439          * and kill any outstanding requests.
2440          */
2441         usb_reset (dev);
2442         for (i = 0; i < dev->n_ep; i++)
2443                 nuke (&dev->ep [i]);
2444
2445         /* report disconnect; the driver is already quiesced */
2446         if (driver) {
2447                 spin_unlock(&dev->lock);
2448                 driver->disconnect(&dev->gadget);
2449                 spin_lock(&dev->lock);
2450         }
2451
2452         usb_reinit (dev);
2453 }
2454
2455 static int net2280_stop(struct usb_gadget *_gadget,
2456                 struct usb_gadget_driver *driver)
2457 {
2458         struct net2280  *dev;
2459         unsigned long   flags;
2460
2461         dev = container_of (_gadget, struct net2280, gadget);
2462
2463         spin_lock_irqsave (&dev->lock, flags);
2464         stop_activity (dev, driver);
2465         spin_unlock_irqrestore (&dev->lock, flags);
2466
2467         dev->driver = NULL;
2468
2469         net2280_led_active (dev, 0);
2470
2471         /* Disable full-speed test mode */
2472         if (dev->pdev->vendor == 0x17cc)
2473                 writel(0, &dev->usb->xcvrdiag);
2474
2475         device_remove_file (&dev->pdev->dev, &dev_attr_function);
2476         device_remove_file (&dev->pdev->dev, &dev_attr_queues);
2477
2478         DEBUG(dev, "unregistered driver '%s'\n",
2479                         driver ? driver->driver.name : "");
2480
2481         return 0;
2482 }
2483
2484 /*-------------------------------------------------------------------------*/
2485
2486 /* handle ep0, ep-e, ep-f with 64 byte packets: packet per irq.
2487  * also works for dma-capable endpoints, in pio mode or just
2488  * to manually advance the queue after short OUT transfers.
2489  */
2490 static void handle_ep_small (struct net2280_ep *ep)
2491 {
2492         struct net2280_request  *req;
2493         u32                     t;
2494         /* 0 error, 1 mid-data, 2 done */
2495         int                     mode = 1;
2496
2497         if (!list_empty (&ep->queue))
2498                 req = list_entry (ep->queue.next,
2499                         struct net2280_request, queue);
2500         else
2501                 req = NULL;
2502
2503         /* ack all, and handle what we care about */
2504         t = readl (&ep->regs->ep_stat);
2505         ep->irqs++;
2506 #if 0
2507         VDEBUG (ep->dev, "%s ack ep_stat %08x, req %p\n",
2508                         ep->ep.name, t, req ? &req->req : 0);
2509 #endif
2510         if (!ep->is_in || ep->dev->pdev->device == 0x2280)
2511                 writel (t & ~(1 << NAK_OUT_PACKETS), &ep->regs->ep_stat);
2512         else
2513                 /* Added for 2282 */
2514                 writel (t, &ep->regs->ep_stat);
2515
2516         /* for ep0, monitor token irqs to catch data stage length errors
2517          * and to synchronize on status.
2518          *
2519          * also, to defer reporting of protocol stalls ... here's where
2520          * data or status first appears, handling stalls here should never
2521          * cause trouble on the host side..
2522          *
2523          * control requests could be slightly faster without token synch for
2524          * status, but status can jam up that way.
2525          */
2526         if (unlikely (ep->num == 0)) {
2527                 if (ep->is_in) {
2528                         /* status; stop NAKing */
2529                         if (t & (1 << DATA_OUT_PING_TOKEN_INTERRUPT)) {
2530                                 if (ep->dev->protocol_stall) {
2531                                         ep->stopped = 1;
2532                                         set_halt (ep);
2533                                 }
2534                                 if (!req)
2535                                         allow_status (ep);
2536                                 mode = 2;
2537                         /* reply to extra IN data tokens with a zlp */
2538                         } else if (t & (1 << DATA_IN_TOKEN_INTERRUPT)) {
2539                                 if (ep->dev->protocol_stall) {
2540                                         ep->stopped = 1;
2541                                         set_halt (ep);
2542                                         mode = 2;
2543                                 } else if (ep->responded &&
2544                                                 !req && !ep->stopped)
2545                                         write_fifo (ep, NULL);
2546                         }
2547                 } else {
2548                         /* status; stop NAKing */
2549                         if (t & (1 << DATA_IN_TOKEN_INTERRUPT)) {
2550                                 if (ep->dev->protocol_stall) {
2551                                         ep->stopped = 1;
2552                                         set_halt (ep);
2553                                 }
2554                                 mode = 2;
2555                         /* an extra OUT token is an error */
2556                         } else if (((t & (1 << DATA_OUT_PING_TOKEN_INTERRUPT))
2557                                         && req
2558                                         && req->req.actual == req->req.length)
2559                                         || (ep->responded && !req)) {
2560                                 ep->dev->protocol_stall = 1;
2561                                 set_halt (ep);
2562                                 ep->stopped = 1;
2563                                 if (req)
2564                                         done (ep, req, -EOVERFLOW);
2565                                 req = NULL;
2566                         }
2567                 }
2568         }
2569
2570         if (unlikely (!req))
2571                 return;
2572
2573         /* manual DMA queue advance after short OUT */
2574         if (likely (ep->dma)) {
2575                 if (t & (1 << SHORT_PACKET_TRANSFERRED_INTERRUPT)) {
2576                         u32     count;
2577                         int     stopped = ep->stopped;
2578
2579                         /* TRANSFERRED works around OUT_DONE erratum 0112.
2580                          * we expect (N <= maxpacket) bytes; host wrote M.
2581                          * iff (M < N) we won't ever see a DMA interrupt.
2582                          */
2583                         ep->stopped = 1;
2584                         for (count = 0; ; t = readl (&ep->regs->ep_stat)) {
2585
2586                                 /* any preceding dma transfers must finish.
2587                                  * dma handles (M >= N), may empty the queue
2588                                  */
2589                                 scan_dma_completions (ep);
2590                                 if (unlikely (list_empty (&ep->queue)
2591                                                 || ep->out_overflow)) {
2592                                         req = NULL;
2593                                         break;
2594                                 }
2595                                 req = list_entry (ep->queue.next,
2596                                         struct net2280_request, queue);
2597
2598                                 /* here either (M < N), a "real" short rx;
2599                                  * or (M == N) and the queue didn't empty
2600                                  */
2601                                 if (likely (t & (1 << FIFO_EMPTY))) {
2602                                         count = readl (&ep->dma->dmacount);
2603                                         count &= DMA_BYTE_COUNT_MASK;
2604                                         if (readl (&ep->dma->dmadesc)
2605                                                         != req->td_dma)
2606                                                 req = NULL;
2607                                         break;
2608                                 }
2609                                 udelay(1);
2610                         }
2611
2612                         /* stop DMA, leave ep NAKing */
2613                         writel ((1 << DMA_ABORT), &ep->dma->dmastat);
2614                         spin_stop_dma (ep->dma);
2615
2616                         if (likely (req)) {
2617                                 req->td->dmacount = 0;
2618                                 t = readl (&ep->regs->ep_avail);
2619                                 dma_done (ep, req, count,
2620                                         (ep->out_overflow || t)
2621                                                 ? -EOVERFLOW : 0);
2622                         }
2623
2624                         /* also flush to prevent erratum 0106 trouble */
2625                         if (unlikely (ep->out_overflow
2626                                         || (ep->dev->chiprev == 0x0100
2627                                                 && ep->dev->gadget.speed
2628                                                         == USB_SPEED_FULL))) {
2629                                 out_flush (ep);
2630                                 ep->out_overflow = 0;
2631                         }
2632
2633                         /* (re)start dma if needed, stop NAKing */
2634                         ep->stopped = stopped;
2635                         if (!list_empty (&ep->queue))
2636                                 restart_dma (ep);
2637                 } else
2638                         DEBUG (ep->dev, "%s dma ep_stat %08x ??\n",
2639                                         ep->ep.name, t);
2640                 return;
2641
2642         /* data packet(s) received (in the fifo, OUT) */
2643         } else if (t & (1 << DATA_PACKET_RECEIVED_INTERRUPT)) {
2644                 if (read_fifo (ep, req) && ep->num != 0)
2645                         mode = 2;
2646
2647         /* data packet(s) transmitted (IN) */
2648         } else if (t & (1 << DATA_PACKET_TRANSMITTED_INTERRUPT)) {
2649                 unsigned        len;
2650
2651                 len = req->req.length - req->req.actual;
2652                 if (len > ep->ep.maxpacket)
2653                         len = ep->ep.maxpacket;
2654                 req->req.actual += len;
2655
2656                 /* if we wrote it all, we're usually done */
2657                 if (req->req.actual == req->req.length) {
2658                         if (ep->num == 0) {
2659                                 /* send zlps until the status stage */
2660                         } else if (!req->req.zero || len != ep->ep.maxpacket)
2661                                 mode = 2;
2662                 }
2663
2664         /* there was nothing to do ...  */
2665         } else if (mode == 1)
2666                 return;
2667
2668         /* done */
2669         if (mode == 2) {
2670                 /* stream endpoints often resubmit/unlink in completion */
2671                 done (ep, req, 0);
2672
2673                 /* maybe advance queue to next request */
2674                 if (ep->num == 0) {
2675                         /* NOTE:  net2280 could let gadget driver start the
2676                          * status stage later. since not all controllers let
2677                          * them control that, the api doesn't (yet) allow it.
2678                          */
2679                         if (!ep->stopped)
2680                                 allow_status (ep);
2681                         req = NULL;
2682                 } else {
2683                         if (!list_empty (&ep->queue) && !ep->stopped)
2684                                 req = list_entry (ep->queue.next,
2685                                         struct net2280_request, queue);
2686                         else
2687                                 req = NULL;
2688                         if (req && !ep->is_in)
2689                                 stop_out_naking (ep);
2690                 }
2691         }
2692
2693         /* is there a buffer for the next packet?
2694          * for best streaming performance, make sure there is one.
2695          */
2696         if (req && !ep->stopped) {
2697
2698                 /* load IN fifo with next packet (may be zlp) */
2699                 if (t & (1 << DATA_PACKET_TRANSMITTED_INTERRUPT))
2700                         write_fifo (ep, &req->req);
2701         }
2702 }
2703
2704 static struct net2280_ep *
2705 get_ep_by_addr (struct net2280 *dev, u16 wIndex)
2706 {
2707         struct net2280_ep       *ep;
2708
2709         if ((wIndex & USB_ENDPOINT_NUMBER_MASK) == 0)
2710                 return &dev->ep [0];
2711         list_for_each_entry (ep, &dev->gadget.ep_list, ep.ep_list) {
2712                 u8      bEndpointAddress;
2713
2714                 if (!ep->desc)
2715                         continue;
2716                 bEndpointAddress = ep->desc->bEndpointAddress;
2717                 if ((wIndex ^ bEndpointAddress) & USB_DIR_IN)
2718                         continue;
2719                 if ((wIndex & 0x0f) == (bEndpointAddress & 0x0f))
2720                         return ep;
2721         }
2722         return NULL;
2723 }
2724
2725 static void defect7374_workaround(struct net2280 *dev, struct usb_ctrlrequest r)
2726 {
2727         u32 scratch, fsmvalue;
2728         u32 ack_wait_timeout, state;
2729
2730         /* Workaround for Defect 7374 (U1/U2 erroneously rejected): */
2731         scratch = get_idx_reg(dev->regs, SCRATCH);
2732         fsmvalue = scratch & (0xf << DEFECT7374_FSM_FIELD);
2733         scratch &= ~(0xf << DEFECT7374_FSM_FIELD);
2734
2735         if (!((fsmvalue == DEFECT7374_FSM_WAITING_FOR_CONTROL_READ) &&
2736                                 (r.bRequestType & USB_DIR_IN)))
2737                 return;
2738
2739         /* This is the first Control Read for this connection: */
2740         if (!(readl(&dev->usb->usbstat) & (1 << SUPER_SPEED_MODE))) {
2741                 /*
2742                  * Connection is NOT SS:
2743                  * - Connection must be FS or HS.
2744                  * - This FSM state should allow workaround software to
2745                  * run after the next USB connection.
2746                  */
2747                 scratch |= DEFECT7374_FSM_NON_SS_CONTROL_READ;
2748                 goto restore_data_eps;
2749         }
2750
2751         /* Connection is SS: */
2752         for (ack_wait_timeout = 0;
2753                         ack_wait_timeout < DEFECT_7374_NUMBEROF_MAX_WAIT_LOOPS;
2754                         ack_wait_timeout++) {
2755
2756                 state = readl(&dev->plregs->pl_ep_status_1)
2757                         & (0xff << STATE);
2758                 if ((state >= (ACK_GOOD_NORMAL << STATE)) &&
2759                         (state <= (ACK_GOOD_MORE_ACKS_TO_COME << STATE))) {
2760                         scratch |= DEFECT7374_FSM_SS_CONTROL_READ;
2761                         break;
2762                 }
2763
2764                 /*
2765                  * We have not yet received host's Data Phase ACK
2766                  * - Wait and try again.
2767                  */
2768                 udelay(DEFECT_7374_PROCESSOR_WAIT_TIME);
2769
2770                 continue;
2771         }
2772
2773
2774         if (ack_wait_timeout >= DEFECT_7374_NUMBEROF_MAX_WAIT_LOOPS) {
2775                 ERROR(dev, "FAIL: Defect 7374 workaround waited but failed");
2776                 ERROR(dev, "to detect SS host's data phase ACK.");
2777                 ERROR(dev, "PL_EP_STATUS_1(23:16):.Expected from 0x11 to 0x16");
2778                 ERROR(dev, "got 0x%2.2x.\n", state >> STATE);
2779         } else {
2780                 WARNING(dev, "INFO: Defect 7374 workaround waited about\n");
2781                 WARNING(dev, "%duSec for Control Read Data Phase ACK\n",
2782                         DEFECT_7374_PROCESSOR_WAIT_TIME * ack_wait_timeout);
2783         }
2784
2785 restore_data_eps:
2786         /*
2787          * Restore data EPs to their pre-workaround settings (disabled,
2788          * initialized, and other details).
2789          */
2790         defect7374_disable_data_eps(dev);
2791
2792         set_idx_reg(dev->regs, SCRATCH, scratch);
2793
2794         return;
2795 }
2796
2797 static void ep_stall(struct net2280_ep *ep, int stall)
2798 {
2799         struct net2280 *dev = ep->dev;
2800         u32 val;
2801         static const u32 ep_pl[9] = { 0, 3, 4, 7, 8, 2, 5, 6, 9 };
2802
2803         if (stall) {
2804                 writel((1 << SET_ENDPOINT_HALT) |
2805                        /* (1 << SET_NAK_PACKETS) | */
2806                        (1 << CLEAR_CONTROL_STATUS_PHASE_HANDSHAKE),
2807                        &ep->regs->ep_rsp);
2808                 ep->is_halt = 1;
2809         } else {
2810                 if (dev->gadget.speed == USB_SPEED_SUPER) {
2811                         /*
2812                          * Workaround for SS SeqNum not cleared via
2813                          * Endpoint Halt (Clear) bit. select endpoint
2814                          */
2815                         val = readl(&dev->plregs->pl_ep_ctrl);
2816                         val = (val & ~0x1f) | ep_pl[ep->num];
2817                         writel(val, &dev->plregs->pl_ep_ctrl);
2818
2819                         val |= (1 << SEQUENCE_NUMBER_RESET);
2820                         writel(val, &dev->plregs->pl_ep_ctrl);
2821                 }
2822                 val = readl(&ep->regs->ep_rsp);
2823                 val |= (1 << CLEAR_ENDPOINT_HALT) |
2824                         (1 << CLEAR_ENDPOINT_TOGGLE);
2825                 writel(val
2826                        /* | (1 << CLEAR_NAK_PACKETS)*/
2827                        , &ep->regs->ep_rsp);
2828                 ep->is_halt = 0;
2829                 val = readl(&ep->regs->ep_rsp);
2830         }
2831 }
2832
2833 static void ep_stdrsp(struct net2280_ep *ep, int value, int wedged)
2834 {
2835         /* set/clear, then synch memory views with the device */
2836         if (value) {
2837                 ep->stopped = 1;
2838                 if (ep->num == 0)
2839                         ep->dev->protocol_stall = 1;
2840                 else {
2841                         if (ep->dma)
2842                                 ep_stop_dma(ep);
2843                         ep_stall(ep, true);
2844                 }
2845
2846                 if (wedged)
2847                         ep->wedged = 1;
2848         } else {
2849                 ep->stopped = 0;
2850                 ep->wedged = 0;
2851
2852                 ep_stall(ep, false);
2853
2854                 /* Flush the queue */
2855                 if (!list_empty(&ep->queue)) {
2856                         struct net2280_request *req =
2857                             list_entry(ep->queue.next, struct net2280_request,
2858                                        queue);
2859                         if (ep->dma)
2860                                 resume_dma(ep);
2861                         else {
2862                                 if (ep->is_in)
2863                                         write_fifo(ep, &req->req);
2864                                 else {
2865                                         if (read_fifo(ep, req))
2866                                                 done(ep, req, 0);
2867                                 }
2868                         }
2869                 }
2870         }
2871 }
2872
2873 static void handle_stat0_irqs_superspeed(struct net2280 *dev,
2874                 struct net2280_ep *ep, struct usb_ctrlrequest r)
2875 {
2876         int tmp = 0;
2877
2878 #define w_value         le16_to_cpu(r.wValue)
2879 #define w_index         le16_to_cpu(r.wIndex)
2880 #define w_length        le16_to_cpu(r.wLength)
2881
2882         switch (r.bRequest) {
2883                 struct net2280_ep *e;
2884                 u16 status;
2885
2886         case USB_REQ_SET_CONFIGURATION:
2887                 dev->addressed_state = !w_value;
2888                 goto usb3_delegate;
2889
2890         case USB_REQ_GET_STATUS:
2891                 switch (r.bRequestType) {
2892                 case (USB_DIR_IN | USB_TYPE_STANDARD | USB_RECIP_DEVICE):
2893                         status = dev->wakeup_enable ? 0x02 : 0x00;
2894                         if (dev->selfpowered)
2895                                 status |= 1 << 0;
2896                         status |= (dev->u1_enable << 2 | dev->u2_enable << 3 |
2897                                                         dev->ltm_enable << 4);
2898                         writel(0, &dev->epregs[0].ep_irqenb);
2899                         set_fifo_bytecount(ep, sizeof(status));
2900                         writel((__force u32) status, &dev->epregs[0].ep_data);
2901                         allow_status_338x(ep);
2902                         break;
2903
2904                 case (USB_DIR_IN | USB_TYPE_STANDARD | USB_RECIP_ENDPOINT):
2905                         e = get_ep_by_addr(dev, w_index);
2906                         if (!e)
2907                                 goto do_stall3;
2908                         status = readl(&e->regs->ep_rsp) &
2909                                                 (1 << CLEAR_ENDPOINT_HALT);
2910                         writel(0, &dev->epregs[0].ep_irqenb);
2911                         set_fifo_bytecount(ep, sizeof(status));
2912                         writel((__force u32) status, &dev->epregs[0].ep_data);
2913                         allow_status_338x(ep);
2914                         break;
2915
2916                 default:
2917                         goto usb3_delegate;
2918                 }
2919                 break;
2920
2921         case USB_REQ_CLEAR_FEATURE:
2922                 switch (r.bRequestType) {
2923                 case (USB_DIR_OUT | USB_TYPE_STANDARD | USB_RECIP_DEVICE):
2924                         if (!dev->addressed_state) {
2925                                 switch (w_value) {
2926                                 case USB_DEVICE_U1_ENABLE:
2927                                         dev->u1_enable = 0;
2928                                         writel(readl(&dev->usb_ext->usbctl2) &
2929                                                 ~(1 << U1_ENABLE),
2930                                                 &dev->usb_ext->usbctl2);
2931                                         allow_status_338x(ep);
2932                                         goto next_endpoints3;
2933
2934                                 case USB_DEVICE_U2_ENABLE:
2935                                         dev->u2_enable = 0;
2936                                         writel(readl(&dev->usb_ext->usbctl2) &
2937                                                 ~(1 << U2_ENABLE),
2938                                                 &dev->usb_ext->usbctl2);
2939                                         allow_status_338x(ep);
2940                                         goto next_endpoints3;
2941
2942                                 case USB_DEVICE_LTM_ENABLE:
2943                                         dev->ltm_enable = 0;
2944                                         writel(readl(&dev->usb_ext->usbctl2) &
2945                                                 ~(1 << LTM_ENABLE),
2946                                                 &dev->usb_ext->usbctl2);
2947                                         allow_status_338x(ep);
2948                                         goto next_endpoints3;
2949
2950                                 default:
2951                                         break;
2952                                 }
2953                         }
2954                         if (w_value == USB_DEVICE_REMOTE_WAKEUP) {
2955                                 dev->wakeup_enable = 0;
2956                                 writel(readl(&dev->usb->usbctl) &
2957                                         ~(1 << DEVICE_REMOTE_WAKEUP_ENABLE),
2958                                         &dev->usb->usbctl);
2959                                 allow_status_338x(ep);
2960                                 break;
2961                         }
2962                         goto usb3_delegate;
2963
2964                 case (USB_DIR_OUT | USB_TYPE_STANDARD | USB_RECIP_ENDPOINT):
2965                         e = get_ep_by_addr(dev, w_index);
2966                         if (!e)
2967                                 goto do_stall3;
2968                         if (w_value != USB_ENDPOINT_HALT)
2969                                 goto do_stall3;
2970                         VDEBUG(dev, "%s clear halt\n", e->ep.name);
2971                         ep_stall(e, false);
2972                         if (!list_empty(&e->queue) && e->td_dma)
2973                                 restart_dma(e);
2974                         allow_status(ep);
2975                         ep->stopped = 1;
2976                         break;
2977
2978                 default:
2979                         goto usb3_delegate;
2980                 }
2981                 break;
2982         case USB_REQ_SET_FEATURE:
2983                 switch (r.bRequestType) {
2984                 case (USB_DIR_OUT | USB_TYPE_STANDARD | USB_RECIP_DEVICE):
2985                         if (!dev->addressed_state) {
2986                                 switch (w_value) {
2987                                 case USB_DEVICE_U1_ENABLE:
2988                                         dev->u1_enable = 1;
2989                                         writel(readl(&dev->usb_ext->usbctl2) |
2990                                                 (1 << U1_ENABLE),
2991                                                 &dev->usb_ext->usbctl2);
2992                                         allow_status_338x(ep);
2993                                         goto next_endpoints3;
2994
2995                                 case USB_DEVICE_U2_ENABLE:
2996                                         dev->u2_enable = 1;
2997                                         writel(readl(&dev->usb_ext->usbctl2) |
2998                                                 (1 << U2_ENABLE),
2999                                                 &dev->usb_ext->usbctl2);
3000                                         allow_status_338x(ep);
3001                                         goto next_endpoints3;
3002
3003                                 case USB_DEVICE_LTM_ENABLE:
3004                                         dev->ltm_enable = 1;
3005                                         writel(readl(&dev->usb_ext->usbctl2) |
3006                                                 (1 << LTM_ENABLE),
3007                                                 &dev->usb_ext->usbctl2);
3008                                         allow_status_338x(ep);
3009                                         goto next_endpoints3;
3010                                 default:
3011                                         break;
3012                                 }
3013                         }
3014
3015                         if (w_value == USB_DEVICE_REMOTE_WAKEUP) {
3016                                 dev->wakeup_enable = 1;
3017                                 writel(readl(&dev->usb->usbctl) |
3018                                         (1 << DEVICE_REMOTE_WAKEUP_ENABLE),
3019                                         &dev->usb->usbctl);
3020                                 allow_status_338x(ep);
3021                                 break;
3022                         }
3023                         goto usb3_delegate;
3024
3025                 case (USB_DIR_OUT | USB_TYPE_STANDARD | USB_RECIP_ENDPOINT):
3026                         e = get_ep_by_addr(dev, w_index);
3027                         if (!e || (w_value != USB_ENDPOINT_HALT))
3028                                 goto do_stall3;
3029                         ep_stdrsp(e, true, false);
3030                         allow_status_338x(ep);
3031                         break;
3032
3033                 default:
3034                         goto usb3_delegate;
3035                 }
3036
3037                 break;
3038         default:
3039
3040 usb3_delegate:
3041                 VDEBUG(dev, "setup %02x.%02x v%04x i%04x l%04x ep_cfg %08x\n",
3042                                 r.bRequestType, r.bRequest,
3043                                 w_value, w_index, w_length,
3044                                 readl(&ep->cfg->ep_cfg));
3045
3046                 ep->responded = 0;
3047                 spin_unlock(&dev->lock);
3048                 tmp = dev->driver->setup(&dev->gadget, &r);
3049                 spin_lock(&dev->lock);
3050         }
3051 do_stall3:
3052         if (tmp < 0) {
3053                 VDEBUG(dev, "req %02x.%02x protocol STALL; stat %d\n",
3054                                 r.bRequestType, r.bRequest, tmp);
3055                 dev->protocol_stall = 1;
3056                 /* TD 9.9 Halt Endpoint test. TD 9.22 Set feature test */
3057                 ep_stall(ep, true);
3058         }
3059
3060 next_endpoints3:
3061
3062 #undef  w_value
3063 #undef  w_index
3064 #undef  w_length
3065
3066         return;
3067 }
3068
3069 static void handle_stat0_irqs (struct net2280 *dev, u32 stat)
3070 {
3071         struct net2280_ep       *ep;
3072         u32                     num, scratch;
3073
3074         /* most of these don't need individual acks */
3075         stat &= ~(1 << INTA_ASSERTED);
3076         if (!stat)
3077                 return;
3078         // DEBUG (dev, "irqstat0 %04x\n", stat);
3079
3080         /* starting a control request? */
3081         if (unlikely (stat & (1 << SETUP_PACKET_INTERRUPT))) {
3082                 union {
3083                         u32                     raw [2];
3084                         struct usb_ctrlrequest  r;
3085                 } u;
3086                 int                             tmp;
3087                 struct net2280_request          *req;
3088
3089                 if (dev->gadget.speed == USB_SPEED_UNKNOWN) {
3090                         u32 val = readl(&dev->usb->usbstat);
3091                         if (val & (1 << SUPER_SPEED)) {
3092                                 dev->gadget.speed = USB_SPEED_SUPER;
3093                                 usb_ep_set_maxpacket_limit(&dev->ep[0].ep,
3094                                                 EP0_SS_MAX_PACKET_SIZE);
3095                         } else if (val & (1 << HIGH_SPEED)) {
3096                                 dev->gadget.speed = USB_SPEED_HIGH;
3097                                 usb_ep_set_maxpacket_limit(&dev->ep[0].ep,
3098                                                 EP0_HS_MAX_PACKET_SIZE);
3099                         } else {
3100                                 dev->gadget.speed = USB_SPEED_FULL;
3101                                 usb_ep_set_maxpacket_limit(&dev->ep[0].ep,
3102                                                 EP0_HS_MAX_PACKET_SIZE);
3103                         }
3104                         net2280_led_speed (dev, dev->gadget.speed);
3105                         DEBUG(dev, "%s\n", usb_speed_string(dev->gadget.speed));
3106                 }
3107
3108                 ep = &dev->ep [0];
3109                 ep->irqs++;
3110
3111                 /* make sure any leftover request state is cleared */
3112                 stat &= ~(1 << ENDPOINT_0_INTERRUPT);
3113                 while (!list_empty (&ep->queue)) {
3114                         req = list_entry (ep->queue.next,
3115                                         struct net2280_request, queue);
3116                         done (ep, req, (req->req.actual == req->req.length)
3117                                                 ? 0 : -EPROTO);
3118                 }
3119                 ep->stopped = 0;
3120                 dev->protocol_stall = 0;
3121                 if (dev->pdev->vendor == 0x10b5)
3122                         ep->is_halt = 0;
3123                 else{
3124                         if (ep->dev->pdev->device == 0x2280)
3125                                 tmp = (1 << FIFO_OVERFLOW) |
3126                                     (1 << FIFO_UNDERFLOW);
3127                         else
3128                                 tmp = 0;
3129
3130                         writel(tmp | (1 << TIMEOUT) |
3131                                    (1 << USB_STALL_SENT) |
3132                                    (1 << USB_IN_NAK_SENT) |
3133                                    (1 << USB_IN_ACK_RCVD) |
3134                                    (1 << USB_OUT_PING_NAK_SENT) |
3135                                    (1 << USB_OUT_ACK_SENT) |
3136                                    (1 << SHORT_PACKET_OUT_DONE_INTERRUPT) |
3137                                    (1 << SHORT_PACKET_TRANSFERRED_INTERRUPT) |
3138                                    (1 << DATA_PACKET_RECEIVED_INTERRUPT) |
3139                                    (1 << DATA_PACKET_TRANSMITTED_INTERRUPT) |
3140                                    (1 << DATA_OUT_PING_TOKEN_INTERRUPT) |
3141                                    (1 << DATA_IN_TOKEN_INTERRUPT)
3142                                    , &ep->regs->ep_stat);
3143                 }
3144                 u.raw[0] = readl(&dev->usb->setup0123);
3145                 u.raw[1] = readl(&dev->usb->setup4567);
3146
3147                 cpu_to_le32s (&u.raw [0]);
3148                 cpu_to_le32s (&u.raw [1]);
3149
3150                 if (dev->pdev->vendor == 0x10b5)
3151                         defect7374_workaround(dev, u.r);
3152
3153                 tmp = 0;
3154
3155 #define w_value         le16_to_cpu(u.r.wValue)
3156 #define w_index         le16_to_cpu(u.r.wIndex)
3157 #define w_length        le16_to_cpu(u.r.wLength)
3158
3159                 /* ack the irq */
3160                 writel (1 << SETUP_PACKET_INTERRUPT, &dev->regs->irqstat0);
3161                 stat ^= (1 << SETUP_PACKET_INTERRUPT);
3162
3163                 /* watch control traffic at the token level, and force
3164                  * synchronization before letting the status stage happen.
3165                  * FIXME ignore tokens we'll NAK, until driver responds.
3166                  * that'll mean a lot less irqs for some drivers.
3167                  */
3168                 ep->is_in = (u.r.bRequestType & USB_DIR_IN) != 0;
3169                 if (ep->is_in) {
3170                         scratch = (1 << DATA_PACKET_TRANSMITTED_INTERRUPT)
3171                                 | (1 << DATA_OUT_PING_TOKEN_INTERRUPT)
3172                                 | (1 << DATA_IN_TOKEN_INTERRUPT);
3173                         stop_out_naking (ep);
3174                 } else
3175                         scratch = (1 << DATA_PACKET_RECEIVED_INTERRUPT)
3176                                 | (1 << DATA_OUT_PING_TOKEN_INTERRUPT)
3177                                 | (1 << DATA_IN_TOKEN_INTERRUPT);
3178                 writel (scratch, &dev->epregs [0].ep_irqenb);
3179
3180                 /* we made the hardware handle most lowlevel requests;
3181                  * everything else goes uplevel to the gadget code.
3182                  */
3183                 ep->responded = 1;
3184
3185                 if (dev->gadget.speed == USB_SPEED_SUPER) {
3186                         handle_stat0_irqs_superspeed(dev, ep, u.r);
3187                         goto next_endpoints;
3188                 }
3189
3190                 switch (u.r.bRequest) {
3191                 case USB_REQ_GET_STATUS: {
3192                         struct net2280_ep       *e;
3193                         __le32                  status;
3194
3195                         /* hw handles device and interface status */
3196                         if (u.r.bRequestType != (USB_DIR_IN|USB_RECIP_ENDPOINT))
3197                                 goto delegate;
3198                         if ((e = get_ep_by_addr (dev, w_index)) == NULL
3199                                         || w_length > 2)
3200                                 goto do_stall;
3201
3202                         if (readl (&e->regs->ep_rsp)
3203                                         & (1 << SET_ENDPOINT_HALT))
3204                                 status = cpu_to_le32 (1);
3205                         else
3206                                 status = cpu_to_le32 (0);
3207
3208                         /* don't bother with a request object! */
3209                         writel (0, &dev->epregs [0].ep_irqenb);
3210                         set_fifo_bytecount (ep, w_length);
3211                         writel ((__force u32)status, &dev->epregs [0].ep_data);
3212                         allow_status (ep);
3213                         VDEBUG (dev, "%s stat %02x\n", ep->ep.name, status);
3214                         goto next_endpoints;
3215                         }
3216                         break;
3217                 case USB_REQ_CLEAR_FEATURE: {
3218                         struct net2280_ep       *e;
3219
3220                         /* hw handles device features */
3221                         if (u.r.bRequestType != USB_RECIP_ENDPOINT)
3222                                 goto delegate;
3223                         if (w_value != USB_ENDPOINT_HALT
3224                                         || w_length != 0)
3225                                 goto do_stall;
3226                         if ((e = get_ep_by_addr (dev, w_index)) == NULL)
3227                                 goto do_stall;
3228                         if (e->wedged) {
3229                                 VDEBUG(dev, "%s wedged, halt not cleared\n",
3230                                                 ep->ep.name);
3231                         } else {
3232                                 VDEBUG(dev, "%s clear halt\n", e->ep.name);
3233                                 clear_halt(e);
3234                                 if (ep->dev->pdev->vendor == 0x10b5 &&
3235                                         !list_empty(&e->queue) && e->td_dma)
3236                                                 restart_dma(e);
3237                         }
3238                         allow_status (ep);
3239                         goto next_endpoints;
3240                         }
3241                         break;
3242                 case USB_REQ_SET_FEATURE: {
3243                         struct net2280_ep       *e;
3244
3245                         /* hw handles device features */
3246                         if (u.r.bRequestType != USB_RECIP_ENDPOINT)
3247                                 goto delegate;
3248                         if (w_value != USB_ENDPOINT_HALT
3249                                         || w_length != 0)
3250                                 goto do_stall;
3251                         if ((e = get_ep_by_addr (dev, w_index)) == NULL)
3252                                 goto do_stall;
3253                         if (e->ep.name == ep0name)
3254                                 goto do_stall;
3255                         set_halt (e);
3256                         if (dev->pdev->vendor == 0x10b5 && e->dma)
3257                                 abort_dma(e);
3258                         allow_status (ep);
3259                         VDEBUG (dev, "%s set halt\n", ep->ep.name);
3260                         goto next_endpoints;
3261                         }
3262                         break;
3263                 default:
3264 delegate:
3265                         VDEBUG (dev, "setup %02x.%02x v%04x i%04x l%04x "
3266                                 "ep_cfg %08x\n",
3267                                 u.r.bRequestType, u.r.bRequest,
3268                                 w_value, w_index, w_length,
3269                                 readl(&ep->cfg->ep_cfg));
3270                         ep->responded = 0;
3271                         spin_unlock (&dev->lock);
3272                         tmp = dev->driver->setup (&dev->gadget, &u.r);
3273                         spin_lock (&dev->lock);
3274                 }
3275
3276                 /* stall ep0 on error */
3277                 if (tmp < 0) {
3278 do_stall:
3279                         VDEBUG (dev, "req %02x.%02x protocol STALL; stat %d\n",
3280                                         u.r.bRequestType, u.r.bRequest, tmp);
3281                         dev->protocol_stall = 1;
3282                 }
3283
3284                 /* some in/out token irq should follow; maybe stall then.
3285                  * driver must queue a request (even zlp) or halt ep0
3286                  * before the host times out.
3287                  */
3288         }
3289
3290 #undef  w_value
3291 #undef  w_index
3292 #undef  w_length
3293
3294 next_endpoints:
3295         /* endpoint data irq ? */
3296         scratch = stat & 0x7f;
3297         stat &= ~0x7f;
3298         for (num = 0; scratch; num++) {
3299                 u32             t;
3300
3301                 /* do this endpoint's FIFO and queue need tending? */
3302                 t = 1 << num;
3303                 if ((scratch & t) == 0)
3304                         continue;
3305                 scratch ^= t;
3306
3307                 ep = &dev->ep [num];
3308                 handle_ep_small (ep);
3309         }
3310
3311         if (stat)
3312                 DEBUG (dev, "unhandled irqstat0 %08x\n", stat);
3313 }
3314
3315 #define DMA_INTERRUPTS ( \
3316                   (1 << DMA_D_INTERRUPT) \
3317                 | (1 << DMA_C_INTERRUPT) \
3318                 | (1 << DMA_B_INTERRUPT) \
3319                 | (1 << DMA_A_INTERRUPT))
3320 #define PCI_ERROR_INTERRUPTS ( \
3321                   (1 << PCI_MASTER_ABORT_RECEIVED_INTERRUPT) \
3322                 | (1 << PCI_TARGET_ABORT_RECEIVED_INTERRUPT) \
3323                 | (1 << PCI_RETRY_ABORT_INTERRUPT))
3324
3325 static void handle_stat1_irqs (struct net2280 *dev, u32 stat)
3326 {
3327         struct net2280_ep       *ep;
3328         u32                     tmp, num, mask, scratch;
3329
3330         /* after disconnect there's nothing else to do! */
3331         tmp = (1 << VBUS_INTERRUPT) | (1 << ROOT_PORT_RESET_INTERRUPT);
3332         mask = (1 << SUPER_SPEED) | (1 << HIGH_SPEED) | (1 << FULL_SPEED);
3333
3334         /* VBUS disconnect is indicated by VBUS_PIN and VBUS_INTERRUPT set.
3335          * Root Port Reset is indicated by ROOT_PORT_RESET_INTERRUPT set and
3336          * both HIGH_SPEED and FULL_SPEED clear (as ROOT_PORT_RESET_INTERRUPT
3337          * only indicates a change in the reset state).
3338          */
3339         if (stat & tmp) {
3340                 writel (tmp, &dev->regs->irqstat1);
3341                 if ((((stat & (1 << ROOT_PORT_RESET_INTERRUPT))
3342                                         && ((readl (&dev->usb->usbstat) & mask)
3343                                                         == 0))
3344                                 || ((readl (&dev->usb->usbctl)
3345                                         & (1 << VBUS_PIN)) == 0)
3346                             ) && ( dev->gadget.speed != USB_SPEED_UNKNOWN)) {
3347                         DEBUG (dev, "disconnect %s\n",
3348                                         dev->driver->driver.name);
3349                         stop_activity (dev, dev->driver);
3350                         ep0_start (dev);
3351                         return;
3352                 }
3353                 stat &= ~tmp;
3354
3355                 /* vBUS can bounce ... one of many reasons to ignore the
3356                  * notion of hotplug events on bus connect/disconnect!
3357                  */
3358                 if (!stat)
3359                         return;
3360         }
3361
3362         /* NOTE: chip stays in PCI D0 state for now, but it could
3363          * enter D1 to save more power
3364          */
3365         tmp = (1 << SUSPEND_REQUEST_CHANGE_INTERRUPT);
3366         if (stat & tmp) {
3367                 writel (tmp, &dev->regs->irqstat1);
3368                 if (stat & (1 << SUSPEND_REQUEST_INTERRUPT)) {
3369                         if (dev->driver->suspend)
3370                                 dev->driver->suspend (&dev->gadget);
3371                         if (!enable_suspend)
3372                                 stat &= ~(1 << SUSPEND_REQUEST_INTERRUPT);
3373                 } else {
3374                         if (dev->driver->resume)
3375                                 dev->driver->resume (&dev->gadget);
3376                         /* at high speed, note erratum 0133 */
3377                 }
3378                 stat &= ~tmp;
3379         }
3380
3381         /* clear any other status/irqs */
3382         if (stat)
3383                 writel (stat, &dev->regs->irqstat1);
3384
3385         /* some status we can just ignore */
3386         if (dev->pdev->device == 0x2280)
3387                 stat &= ~((1 << CONTROL_STATUS_INTERRUPT)
3388                           | (1 << SUSPEND_REQUEST_INTERRUPT)
3389                           | (1 << RESUME_INTERRUPT)
3390                           | (1 << SOF_INTERRUPT));
3391         else
3392                 stat &= ~((1 << CONTROL_STATUS_INTERRUPT)
3393                           | (1 << RESUME_INTERRUPT)
3394                           | (1 << SOF_DOWN_INTERRUPT)
3395                           | (1 << SOF_INTERRUPT));
3396
3397         if (!stat)
3398                 return;
3399         // DEBUG (dev, "irqstat1 %08x\n", stat);
3400
3401         /* DMA status, for ep-{a,b,c,d} */
3402         scratch = stat & DMA_INTERRUPTS;
3403         stat &= ~DMA_INTERRUPTS;
3404         scratch >>= 9;
3405         for (num = 0; scratch; num++) {
3406                 struct net2280_dma_regs __iomem *dma;
3407
3408                 tmp = 1 << num;
3409                 if ((tmp & scratch) == 0)
3410                         continue;
3411                 scratch ^= tmp;
3412
3413                 ep = &dev->ep [num + 1];
3414                 dma = ep->dma;
3415
3416                 if (!dma)
3417                         continue;
3418
3419                 /* clear ep's dma status */
3420                 tmp = readl (&dma->dmastat);
3421                 writel (tmp, &dma->dmastat);
3422
3423                 /* dma sync*/
3424                 if (dev->pdev->vendor == 0x10b5) {
3425                         u32 r_dmacount = readl(&dma->dmacount);
3426                         if (!ep->is_in &&  (r_dmacount & 0x00FFFFFF) &&
3427                             (tmp & (1 << DMA_TRANSACTION_DONE_INTERRUPT)))
3428                                 continue;
3429                 }
3430
3431                 /* chaining should stop on abort, short OUT from fifo,
3432                  * or (stat0 codepath) short OUT transfer.
3433                  */
3434                 if (!use_dma_chaining) {
3435                         if (!(tmp & (1 << DMA_TRANSACTION_DONE_INTERRUPT))) {
3436                                 DEBUG (ep->dev, "%s no xact done? %08x\n",
3437                                         ep->ep.name, tmp);
3438                                 continue;
3439                         }
3440                         stop_dma (ep->dma);
3441                 }
3442
3443                 /* OUT transfers terminate when the data from the
3444                  * host is in our memory.  Process whatever's done.
3445                  * On this path, we know transfer's last packet wasn't
3446                  * less than req->length. NAK_OUT_PACKETS may be set,
3447                  * or the FIFO may already be holding new packets.
3448                  *
3449                  * IN transfers can linger in the FIFO for a very
3450                  * long time ... we ignore that for now, accounting
3451                  * precisely (like PIO does) needs per-packet irqs
3452                  */
3453                 scan_dma_completions (ep);
3454
3455                 /* disable dma on inactive queues; else maybe restart */
3456                 if (list_empty (&ep->queue)) {
3457                         if (use_dma_chaining)
3458                                 stop_dma (ep->dma);
3459                 } else {
3460                         tmp = readl (&dma->dmactl);
3461                         if (!use_dma_chaining
3462                                         || (tmp & (1 << DMA_ENABLE)) == 0)
3463                                 restart_dma (ep);
3464                         else if (ep->is_in && use_dma_chaining) {
3465                                 struct net2280_request  *req;
3466                                 __le32                  dmacount;
3467
3468                                 /* the descriptor at the head of the chain
3469                                  * may still have VALID_BIT clear; that's
3470                                  * used to trigger changing DMA_FIFO_VALIDATE
3471                                  * (affects automagic zlp writes).
3472                                  */
3473                                 req = list_entry (ep->queue.next,
3474                                                 struct net2280_request, queue);
3475                                 dmacount = req->td->dmacount;
3476                                 dmacount &= cpu_to_le32 (
3477                                                 (1 << VALID_BIT)
3478                                                 | DMA_BYTE_COUNT_MASK);
3479                                 if (dmacount && (dmacount & valid_bit) == 0)
3480                                         restart_dma (ep);
3481                         }
3482                 }
3483                 ep->irqs++;
3484         }
3485
3486         /* NOTE:  there are other PCI errors we might usefully notice.
3487          * if they appear very often, here's where to try recovering.
3488          */
3489         if (stat & PCI_ERROR_INTERRUPTS) {
3490                 ERROR (dev, "pci dma error; stat %08x\n", stat);
3491                 stat &= ~PCI_ERROR_INTERRUPTS;
3492                 /* these are fatal errors, but "maybe" they won't
3493                  * happen again ...
3494                  */
3495                 stop_activity (dev, dev->driver);
3496                 ep0_start (dev);
3497                 stat = 0;
3498         }
3499
3500         if (stat)
3501                 DEBUG (dev, "unhandled irqstat1 %08x\n", stat);
3502 }
3503
3504 static irqreturn_t net2280_irq (int irq, void *_dev)
3505 {
3506         struct net2280          *dev = _dev;
3507
3508         /* shared interrupt, not ours */
3509         if (dev->pdev->vendor == 0x17cc &&
3510                 (!(readl(&dev->regs->irqstat0) & (1 << INTA_ASSERTED))))
3511                 return IRQ_NONE;
3512
3513         spin_lock (&dev->lock);
3514
3515         /* handle disconnect, dma, and more */
3516         handle_stat1_irqs (dev, readl (&dev->regs->irqstat1));
3517
3518         /* control requests and PIO */
3519         handle_stat0_irqs (dev, readl (&dev->regs->irqstat0));
3520
3521         if (dev->pdev->vendor == 0x10b5) {
3522                 /* re-enable interrupt to trigger any possible new interrupt */
3523                 u32 pciirqenb1 = readl(&dev->regs->pciirqenb1);
3524                 writel(pciirqenb1 & 0x7FFFFFFF, &dev->regs->pciirqenb1);
3525                 writel(pciirqenb1, &dev->regs->pciirqenb1);
3526         }
3527
3528         spin_unlock (&dev->lock);
3529
3530         return IRQ_HANDLED;
3531 }
3532
3533 /*-------------------------------------------------------------------------*/
3534
3535 static void gadget_release (struct device *_dev)
3536 {
3537         struct net2280  *dev = dev_get_drvdata (_dev);
3538
3539         kfree (dev);
3540 }
3541
3542 /* tear down the binding between this driver and the pci device */
3543
3544 static void net2280_remove (struct pci_dev *pdev)
3545 {
3546         struct net2280          *dev = pci_get_drvdata (pdev);
3547
3548         usb_del_gadget_udc(&dev->gadget);
3549
3550         BUG_ON(dev->driver);
3551
3552         /* then clean up the resources we allocated during probe() */
3553         net2280_led_shutdown (dev);
3554         if (dev->requests) {
3555                 int             i;
3556                 for (i = 1; i < 5; i++) {
3557                         if (!dev->ep [i].dummy)
3558                                 continue;
3559                         pci_pool_free (dev->requests, dev->ep [i].dummy,
3560                                         dev->ep [i].td_dma);
3561                 }
3562                 pci_pool_destroy (dev->requests);
3563         }
3564         if (dev->got_irq)
3565                 free_irq (pdev->irq, dev);
3566         if (use_msi && dev->pdev->vendor == 0x10b5)
3567                 pci_disable_msi(pdev);
3568         if (dev->regs)
3569                 iounmap (dev->regs);
3570         if (dev->region)
3571                 release_mem_region (pci_resource_start (pdev, 0),
3572                                 pci_resource_len (pdev, 0));
3573         if (dev->enabled)
3574                 pci_disable_device (pdev);
3575         device_remove_file (&pdev->dev, &dev_attr_registers);
3576
3577         INFO (dev, "unbind\n");
3578 }
3579
3580 /* wrap this driver around the specified device, but
3581  * don't respond over USB until a gadget driver binds to us.
3582  */
3583
3584 static int net2280_probe (struct pci_dev *pdev, const struct pci_device_id *id)
3585 {
3586         struct net2280          *dev;
3587         unsigned long           resource, len;
3588         void                    __iomem *base = NULL;
3589         int                     retval, i;
3590
3591         /* alloc, and start init */
3592         dev = kzalloc (sizeof *dev, GFP_KERNEL);
3593         if (dev == NULL){
3594                 retval = -ENOMEM;
3595                 goto done;
3596         }
3597
3598         pci_set_drvdata (pdev, dev);
3599         spin_lock_init (&dev->lock);
3600         dev->pdev = pdev;
3601         dev->gadget.ops = &net2280_ops;
3602         dev->gadget.max_speed = (dev->pdev->vendor == 0x10b5) ?
3603                                 USB_SPEED_SUPER : USB_SPEED_HIGH;
3604
3605         /* the "gadget" abstracts/virtualizes the controller */
3606         dev->gadget.name = driver_name;
3607
3608         /* now all the pci goodies ... */
3609         if (pci_enable_device (pdev) < 0) {
3610                 retval = -ENODEV;
3611                 goto done;
3612         }
3613         dev->enabled = 1;
3614
3615         /* BAR 0 holds all the registers
3616          * BAR 1 is 8051 memory; unused here (note erratum 0103)
3617          * BAR 2 is fifo memory; unused here
3618          */
3619         resource = pci_resource_start (pdev, 0);
3620         len = pci_resource_len (pdev, 0);
3621         if (!request_mem_region (resource, len, driver_name)) {
3622                 DEBUG (dev, "controller already in use\n");
3623                 retval = -EBUSY;
3624                 goto done;
3625         }
3626         dev->region = 1;
3627
3628         /* FIXME provide firmware download interface to put
3629          * 8051 code into the chip, e.g. to turn on PCI PM.
3630          */
3631
3632         base = ioremap_nocache (resource, len);
3633         if (base == NULL) {
3634                 DEBUG (dev, "can't map memory\n");
3635                 retval = -EFAULT;
3636                 goto done;
3637         }
3638         dev->regs = (struct net2280_regs __iomem *) base;
3639         dev->usb = (struct net2280_usb_regs __iomem *) (base + 0x0080);
3640         dev->pci = (struct net2280_pci_regs __iomem *) (base + 0x0100);
3641         dev->dma = (struct net2280_dma_regs __iomem *) (base + 0x0180);
3642         dev->dep = (struct net2280_dep_regs __iomem *) (base + 0x0200);
3643         dev->epregs = (struct net2280_ep_regs __iomem *) (base + 0x0300);
3644
3645         if (dev->pdev->vendor == 0x10b5) {
3646                 u32 fsmvalue;
3647                 u32 usbstat;
3648                 dev->usb_ext = (struct usb338x_usb_ext_regs __iomem *)
3649                                                         (base + 0x00b4);
3650                 dev->fiforegs = (struct usb338x_fifo_regs __iomem *)
3651                                                         (base + 0x0500);
3652                 dev->llregs = (struct usb338x_ll_regs __iomem *)
3653                                                         (base + 0x0700);
3654                 dev->ll_lfps_regs = (struct usb338x_ll_lfps_regs __iomem *)
3655                                                         (base + 0x0748);
3656                 dev->ll_tsn_regs = (struct usb338x_ll_tsn_regs __iomem *)
3657                                                         (base + 0x077c);
3658                 dev->ll_chicken_reg = (struct usb338x_ll_chi_regs __iomem *)
3659                                                         (base + 0x079c);
3660                 dev->plregs = (struct usb338x_pl_regs __iomem *)
3661                                                         (base + 0x0800);
3662                 usbstat = readl(&dev->usb->usbstat);
3663                 dev->enhanced_mode = (usbstat & (1 << 11)) ? 1 : 0;
3664                 dev->n_ep = (dev->enhanced_mode) ? 9 : 5;
3665                 /* put into initial config, link up all endpoints */
3666                 fsmvalue = get_idx_reg(dev->regs, SCRATCH) &
3667                                         (0xf << DEFECT7374_FSM_FIELD);
3668                 /* See if firmware needs to set up for workaround: */
3669                 if (fsmvalue == DEFECT7374_FSM_SS_CONTROL_READ)
3670                         writel(0, &dev->usb->usbctl);
3671         } else{
3672                 dev->enhanced_mode = 0;
3673                 dev->n_ep = 7;
3674                 /* put into initial config, link up all endpoints */
3675                 writel(0, &dev->usb->usbctl);
3676         }
3677
3678         usb_reset (dev);
3679         usb_reinit (dev);
3680
3681         /* irq setup after old hardware is cleaned up */
3682         if (!pdev->irq) {
3683                 ERROR (dev, "No IRQ.  Check PCI setup!\n");
3684                 retval = -ENODEV;
3685                 goto done;
3686         }
3687
3688         if (use_msi && dev->pdev->vendor == 0x10b5)
3689                 if (pci_enable_msi(pdev))
3690                         ERROR(dev, "Failed to enable MSI mode\n");
3691
3692         if (request_irq (pdev->irq, net2280_irq, IRQF_SHARED, driver_name, dev)
3693                         != 0) {
3694                 ERROR (dev, "request interrupt %d failed\n", pdev->irq);
3695                 retval = -EBUSY;
3696                 goto done;
3697         }
3698         dev->got_irq = 1;
3699
3700         /* DMA setup */
3701         /* NOTE:  we know only the 32 LSBs of dma addresses may be nonzero */
3702         dev->requests = pci_pool_create ("requests", pdev,
3703                 sizeof (struct net2280_dma),
3704                 0 /* no alignment requirements */,
3705                 0 /* or page-crossing issues */);
3706         if (!dev->requests) {
3707                 DEBUG (dev, "can't get request pool\n");
3708                 retval = -ENOMEM;
3709                 goto done;
3710         }
3711         for (i = 1; i < 5; i++) {
3712                 struct net2280_dma      *td;
3713
3714                 td = pci_pool_alloc (dev->requests, GFP_KERNEL,
3715                                 &dev->ep [i].td_dma);
3716                 if (!td) {
3717                         DEBUG (dev, "can't get dummy %d\n", i);
3718                         retval = -ENOMEM;
3719                         goto done;
3720                 }
3721                 td->dmacount = 0;       /* not VALID */
3722                 td->dmadesc = td->dmaaddr;
3723                 dev->ep [i].dummy = td;
3724         }
3725
3726         /* enable lower-overhead pci memory bursts during DMA */
3727         if (dev->pdev->vendor == 0x17cc)
3728                 writel((1 << DMA_MEMORY_WRITE_AND_INVALIDATE_ENABLE)
3729                         // 256 write retries may not be enough...
3730                         // | (1 << PCI_RETRY_ABORT_ENABLE)
3731                         | (1 << DMA_READ_MULTIPLE_ENABLE)
3732                         | (1 << DMA_READ_LINE_ENABLE)
3733                         , &dev->pci->pcimstctl);
3734         /* erratum 0115 shouldn't appear: Linux inits PCI_LATENCY_TIMER */
3735         pci_set_master (pdev);
3736         pci_try_set_mwi (pdev);
3737
3738         /* ... also flushes any posted pci writes */
3739         dev->chiprev = get_idx_reg (dev->regs, REG_CHIPREV) & 0xffff;
3740
3741         /* done */
3742         INFO (dev, "%s\n", driver_desc);
3743         INFO (dev, "irq %d, pci mem %p, chip rev %04x\n",
3744                         pdev->irq, base, dev->chiprev);
3745         INFO(dev, "version: " DRIVER_VERSION "; dma %s %s\n",
3746                 use_dma ? (use_dma_chaining ? "chaining" : "enabled")
3747                         : "disabled",
3748                 dev->enhanced_mode ? "enhanced mode" : "legacy mode");
3749         retval = device_create_file (&pdev->dev, &dev_attr_registers);
3750         if (retval) goto done;
3751
3752         retval = usb_add_gadget_udc_release(&pdev->dev, &dev->gadget,
3753                         gadget_release);
3754         if (retval)
3755                 goto done;
3756         return 0;
3757
3758 done:
3759         if (dev)
3760                 net2280_remove (pdev);
3761         return retval;
3762 }
3763
3764 /* make sure the board is quiescent; otherwise it will continue
3765  * generating IRQs across the upcoming reboot.
3766  */
3767
3768 static void net2280_shutdown (struct pci_dev *pdev)
3769 {
3770         struct net2280          *dev = pci_get_drvdata (pdev);
3771
3772         /* disable IRQs */
3773         writel (0, &dev->regs->pciirqenb0);
3774         writel (0, &dev->regs->pciirqenb1);
3775
3776         /* disable the pullup so the host will think we're gone */
3777         writel (0, &dev->usb->usbctl);
3778
3779         /* Disable full-speed test mode */
3780         if (dev->pdev->vendor == 0x17cc)
3781                 writel(0, &dev->usb->xcvrdiag);
3782 }
3783
3784
3785 /*-------------------------------------------------------------------------*/
3786
3787 static const struct pci_device_id pci_ids [] = { {
3788         .class =        ((PCI_CLASS_SERIAL_USB << 8) | 0xfe),
3789         .class_mask =   ~0,
3790         .vendor =       0x17cc,
3791         .device =       0x2280,
3792         .subvendor =    PCI_ANY_ID,
3793         .subdevice =    PCI_ANY_ID,
3794 }, {
3795         .class =        ((PCI_CLASS_SERIAL_USB << 8) | 0xfe),
3796         .class_mask =   ~0,
3797         .vendor =       0x17cc,
3798         .device =       0x2282,
3799         .subvendor =    PCI_ANY_ID,
3800         .subdevice =    PCI_ANY_ID,
3801 },
3802         {
3803          .class = ((PCI_CLASS_SERIAL_USB << 8) | 0xfe),
3804          .class_mask = ~0,
3805          .vendor = 0x10b5,
3806          .device = 0x3380,
3807          .subvendor = PCI_ANY_ID,
3808          .subdevice = PCI_ANY_ID,
3809          },
3810         {
3811          .class = ((PCI_CLASS_SERIAL_USB << 8) | 0xfe),
3812          .class_mask = ~0,
3813          .vendor = 0x10b5,
3814          .device = 0x3382,
3815          .subvendor = PCI_ANY_ID,
3816          .subdevice = PCI_ANY_ID,
3817          },
3818 { /* end: all zeroes */ }
3819 };
3820 MODULE_DEVICE_TABLE (pci, pci_ids);
3821
3822 /* pci driver glue; this is a "new style" PCI driver module */
3823 static struct pci_driver net2280_pci_driver = {
3824         .name =         (char *) driver_name,
3825         .id_table =     pci_ids,
3826
3827         .probe =        net2280_probe,
3828         .remove =       net2280_remove,
3829         .shutdown =     net2280_shutdown,
3830
3831         /* FIXME add power management support */
3832 };
3833
3834 MODULE_DESCRIPTION (DRIVER_DESC);
3835 MODULE_AUTHOR ("David Brownell");
3836 MODULE_LICENSE ("GPL");
3837
3838 static int __init init (void)
3839 {
3840         if (!use_dma)
3841                 use_dma_chaining = 0;
3842         return pci_register_driver (&net2280_pci_driver);
3843 }
3844 module_init (init);
3845
3846 static void __exit cleanup (void)
3847 {
3848         pci_unregister_driver (&net2280_pci_driver);
3849 }
3850 module_exit (cleanup);