drivers/net/e1000e/netdev.c

   1 /*******************************************************************************
   2
   3   Intel PRO/1000 Linux driver
   4   Copyright(c) 1999 - 2011 Intel Corporation.
   5
   6   This program is free software; you can redistribute it and/or modify it
   7   under the terms and conditions of the GNU General Public License,
   8   version 2, as published by the Free Software Foundation.
   9
  10   This program is distributed in the hope it will be useful, but WITHOUT
  11   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12   FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13   more details.
  14
  15   You should have received a copy of the GNU General Public License along with
  16   this program; if not, write to the Free Software Foundation, Inc.,
  17   51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  18
  19   The full GNU General Public License is included in this distribution in
  20   the file called "COPYING".
  21
  22   Contact Information:
  23   Linux NICS <linux.nics@intel.com>
  24   e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  25   Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  26
  27 *******************************************************************************/
  28
  29 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  30
  31 #include <linux/module.h>
  32 #include <linux/types.h>
  33 #include <linux/init.h>
  34 #include <linux/pci.h>
  35 #include <linux/vmalloc.h>
  36 #include <linux/pagemap.h>
  37 #include <linux/delay.h>
  38 #include <linux/netdevice.h>
  39 #include <linux/tcp.h>
  40 #include <linux/ipv6.h>
  41 #include <linux/slab.h>
  42 #include <net/checksum.h>
  43 #include <net/ip6_checksum.h>
  44 #include <linux/mii.h>
  45 #include <linux/ethtool.h>
  46 #include <linux/if_vlan.h>
  47 #include <linux/cpu.h>
  48 #include <linux/smp.h>
  49 #include <linux/pm_qos_params.h>
  50 #include <linux/pm_runtime.h>
  51 #include <linux/aer.h>
  52
  53 #include "e1000.h"
  54
  55 #define DRV_EXTRAVERSION "-k2"
  56
  57 #define DRV_VERSION "1.3.10" DRV_EXTRAVERSION
  58 char e1000e_driver_name[] = "e1000e";
  59 const char e1000e_driver_version[] = DRV_VERSION;
  60
  61 static void e1000e_disable_aspm(struct pci_dev *pdev, u16 state);
  62
  63 static const struct e1000_info *e1000_info_tbl[] = {
  64         [board_82571]           = &e1000_82571_info,
  65         [board_82572]           = &e1000_82572_info,
  66         [board_82573]           = &e1000_82573_info,
  67         [board_82574]           = &e1000_82574_info,
  68         [board_82583]           = &e1000_82583_info,
  69         [board_80003es2lan]     = &e1000_es2_info,
  70         [board_ich8lan]         = &e1000_ich8_info,
  71         [board_ich9lan]         = &e1000_ich9_info,
  72         [board_ich10lan]        = &e1000_ich10_info,
  73         [board_pchlan]          = &e1000_pch_info,
  74         [board_pch2lan]         = &e1000_pch2_info,
  75 };
  76
  77 struct e1000_reg_info {
  78         u32 ofs;
  79         char *name;
  80 };
  81
  82 #define E1000_RDFH      0x02410 /* Rx Data FIFO Head - RW */
  83 #define E1000_RDFT      0x02418 /* Rx Data FIFO Tail - RW */
  84 #define E1000_RDFHS     0x02420 /* Rx Data FIFO Head Saved - RW */
  85 #define E1000_RDFTS     0x02428 /* Rx Data FIFO Tail Saved - RW */
  86 #define E1000_RDFPC     0x02430 /* Rx Data FIFO Packet Count - RW */
  87
  88 #define E1000_TDFH      0x03410 /* Tx Data FIFO Head - RW */
  89 #define E1000_TDFT      0x03418 /* Tx Data FIFO Tail - RW */
  90 #define E1000_TDFHS     0x03420 /* Tx Data FIFO Head Saved - RW */
  91 #define E1000_TDFTS     0x03428 /* Tx Data FIFO Tail Saved - RW */
  92 #define E1000_TDFPC     0x03430 /* Tx Data FIFO Packet Count - RW */
  93
  94 static const struct e1000_reg_info e1000_reg_info_tbl[] = {
  95
  96         /* General Registers */
  97         {E1000_CTRL, "CTRL"},
  98         {E1000_STATUS, "STATUS"},
  99         {E1000_CTRL_EXT, "CTRL_EXT"},
 100
 101         /* Interrupt Registers */
 102         {E1000_ICR, "ICR"},
 103
 104         /* Rx Registers */
 105         {E1000_RCTL, "RCTL"},
 106         {E1000_RDLEN, "RDLEN"},
 107         {E1000_RDH, "RDH"},
 108         {E1000_RDT, "RDT"},
 109         {E1000_RDTR, "RDTR"},
 110         {E1000_RXDCTL(0), "RXDCTL"},
 111         {E1000_ERT, "ERT"},
 112         {E1000_RDBAL, "RDBAL"},
 113         {E1000_RDBAH, "RDBAH"},
 114         {E1000_RDFH, "RDFH"},
 115         {E1000_RDFT, "RDFT"},
 116         {E1000_RDFHS, "RDFHS"},
 117         {E1000_RDFTS, "RDFTS"},
 118         {E1000_RDFPC, "RDFPC"},
 119
 120         /* Tx Registers */
 121         {E1000_TCTL, "TCTL"},
 122         {E1000_TDBAL, "TDBAL"},
 123         {E1000_TDBAH, "TDBAH"},
 124         {E1000_TDLEN, "TDLEN"},
 125         {E1000_TDH, "TDH"},
 126         {E1000_TDT, "TDT"},
 127         {E1000_TIDV, "TIDV"},
 128         {E1000_TXDCTL(0), "TXDCTL"},
 129         {E1000_TADV, "TADV"},
 130         {E1000_TARC(0), "TARC"},
 131         {E1000_TDFH, "TDFH"},
 132         {E1000_TDFT, "TDFT"},
 133         {E1000_TDFHS, "TDFHS"},
 134         {E1000_TDFTS, "TDFTS"},
 135         {E1000_TDFPC, "TDFPC"},
 136
 137         /* List Terminator */
 138         {}
 139 };
 140
 141 /*
 142  * e1000_regdump - register printout routine
 143  */
 144 static void e1000_regdump(struct e1000_hw *hw, struct e1000_reg_info *reginfo)
 145 {
 146         int n = 0;
 147         char rname[16];
 148         u32 regs[8];
 149
 150         switch (reginfo->ofs) {
 151         case E1000_RXDCTL(0):
 152                 for (n = 0; n < 2; n++)
 153                         regs[n] = __er32(hw, E1000_RXDCTL(n));
 154                 break;
 155         case E1000_TXDCTL(0):
 156                 for (n = 0; n < 2; n++)
 157                         regs[n] = __er32(hw, E1000_TXDCTL(n));
 158                 break;
 159         case E1000_TARC(0):
 160                 for (n = 0; n < 2; n++)
 161                         regs[n] = __er32(hw, E1000_TARC(n));
 162                 break;
 163         default:
 164                 printk(KERN_INFO "%-15s %08x\n",
 165                        reginfo->name, __er32(hw, reginfo->ofs));
 166                 return;
 167         }
 168
 169         snprintf(rname, 16, "%s%s", reginfo->name, "[0-1]");
 170         printk(KERN_INFO "%-15s ", rname);
 171         for (n = 0; n < 2; n++)
 172                 printk(KERN_CONT "%08x ", regs[n]);
 173         printk(KERN_CONT "\n");
 174 }
 175
 176 /*
 177  * e1000e_dump - Print registers, Tx-ring and Rx-ring
 178  */
 179 static void e1000e_dump(struct e1000_adapter *adapter)
 180 {
 181         struct net_device *netdev = adapter->netdev;
 182         struct e1000_hw *hw = &adapter->hw;
 183         struct e1000_reg_info *reginfo;
 184         struct e1000_ring *tx_ring = adapter->tx_ring;
 185         struct e1000_tx_desc *tx_desc;
 186         struct my_u0 {
 187                 u64 a;
 188                 u64 b;
 189         } *u0;
 190         struct e1000_buffer *buffer_info;
 191         struct e1000_ring *rx_ring = adapter->rx_ring;
 192         union e1000_rx_desc_packet_split *rx_desc_ps;
 193         struct e1000_rx_desc *rx_desc;
 194         struct my_u1 {
 195                 u64 a;
 196                 u64 b;
 197                 u64 c;
 198                 u64 d;
 199         } *u1;
 200         u32 staterr;
 201         int i = 0;
 202
 203         if (!netif_msg_hw(adapter))
 204                 return;
 205
 206         /* Print netdevice Info */
 207         if (netdev) {
 208                 dev_info(&adapter->pdev->dev, "Net device Info\n");
 209                 printk(KERN_INFO "Device Name     state            "
 210                        "trans_start      last_rx\n");
 211                 printk(KERN_INFO "%-15s %016lX %016lX %016lX\n",
 212                        netdev->name, netdev->state, netdev->trans_start,
 213                        netdev->last_rx);
 214         }
 215
 216         /* Print Registers */
 217         dev_info(&adapter->pdev->dev, "Register Dump\n");
 218         printk(KERN_INFO " Register Name   Value\n");
 219         for (reginfo = (struct e1000_reg_info *)e1000_reg_info_tbl;
 220              reginfo->name; reginfo++) {
 221                 e1000_regdump(hw, reginfo);
 222         }
 223
 224         /* Print Tx Ring Summary */
 225         if (!netdev || !netif_running(netdev))
 226                 goto exit;
 227
 228         dev_info(&adapter->pdev->dev, "Tx Ring Summary\n");
 229         printk(KERN_INFO "Queue [NTU] [NTC] [bi(ntc)->dma  ]"
 230                " leng ntw timestamp\n");
 231         buffer_info = &tx_ring->buffer_info[tx_ring->next_to_clean];
 232         printk(KERN_INFO " %5d %5X %5X %016llX %04X %3X %016llX\n",
 233                0, tx_ring->next_to_use, tx_ring->next_to_clean,
 234                (unsigned long long)buffer_info->dma,
 235                buffer_info->length,
 236                buffer_info->next_to_watch,
 237                (unsigned long long)buffer_info->time_stamp);
 238
 239         /* Print Tx Ring */
 240         if (!netif_msg_tx_done(adapter))
 241                 goto rx_ring_summary;
 242
 243         dev_info(&adapter->pdev->dev, "Tx Ring Dump\n");
 244
 245         /* Transmit Descriptor Formats - DEXT[29] is 0 (Legacy) or 1 (Extended)
 246          *
 247          * Legacy Transmit Descriptor
 248          *   +--------------------------------------------------------------+
 249          * 0 |         Buffer Address [63:0] (Reserved on Write Back)       |
 250          *   +--------------------------------------------------------------+
 251          * 8 | Special  |    CSS     | Status |  CMD    |  CSO   |  Length  |
 252          *   +--------------------------------------------------------------+
 253          *   63       48 47        36 35    32 31     24 23    16 15        0
 254          *
 255          * Extended Context Descriptor (DTYP=0x0) for TSO or checksum offload
 256          *   63      48 47    40 39       32 31             16 15    8 7      0
 257          *   +----------------------------------------------------------------+
 258          * 0 |  TUCSE  | TUCS0  |   TUCSS   |     IPCSE       | IPCS0 | IPCSS |
 259          *   +----------------------------------------------------------------+
 260          * 8 |   MSS   | HDRLEN | RSV | STA | TUCMD | DTYP |      PAYLEN      |
 261          *   +----------------------------------------------------------------+
 262          *   63      48 47    40 39 36 35 32 31   24 23  20 19                0
 263          *
 264          * Extended Data Descriptor (DTYP=0x1)
 265          *   +----------------------------------------------------------------+
 266          * 0 |                     Buffer Address [63:0]                      |
 267          *   +----------------------------------------------------------------+
 268          * 8 | VLAN tag |  POPTS  | Rsvd | Status | Command | DTYP |  DTALEN  |
 269          *   +----------------------------------------------------------------+
 270          *   63       48 47     40 39  36 35    32 31     24 23  20 19        0
 271          */
 272         printk(KERN_INFO "Tl[desc]     [address 63:0  ] [SpeCssSCmCsLen]"
 273                " [bi->dma       ] leng  ntw timestamp        bi->skb "
 274                "<-- Legacy format\n");
 275         printk(KERN_INFO "Tc[desc]     [Ce CoCsIpceCoS] [MssHlRSCm0Plen]"
 276                " [bi->dma       ] leng  ntw timestamp        bi->skb "
 277                "<-- Ext Context format\n");
 278         printk(KERN_INFO "Td[desc]     [address 63:0  ] [VlaPoRSCm1Dlen]"
 279                " [bi->dma       ] leng  ntw timestamp        bi->skb "
 280                "<-- Ext Data format\n");
 281         for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
 282                 tx_desc = E1000_TX_DESC(*tx_ring, i);
 283                 buffer_info = &tx_ring->buffer_info[i];
 284                 u0 = (struct my_u0 *)tx_desc;
 285                 printk(KERN_INFO "T%c[0x%03X]    %016llX %016llX %016llX "
 286                        "%04X  %3X %016llX %p",
 287                        (!(le64_to_cpu(u0->b) & (1 << 29)) ? 'l' :
 288                         ((le64_to_cpu(u0->b) & (1 << 20)) ? 'd' : 'c')), i,
 289                        (unsigned long long)le64_to_cpu(u0->a),
 290                        (unsigned long long)le64_to_cpu(u0->b),
 291                        (unsigned long long)buffer_info->dma,
 292                        buffer_info->length, buffer_info->next_to_watch,
 293                        (unsigned long long)buffer_info->time_stamp,
 294                        buffer_info->skb);
 295                 if (i == tx_ring->next_to_use && i == tx_ring->next_to_clean)
 296                         printk(KERN_CONT " NTC/U\n");
 297                 else if (i == tx_ring->next_to_use)
 298                         printk(KERN_CONT " NTU\n");
 299                 else if (i == tx_ring->next_to_clean)
 300                         printk(KERN_CONT " NTC\n");
 301                 else
 302                         printk(KERN_CONT "\n");
 303
 304                 if (netif_msg_pktdata(adapter) && buffer_info->dma != 0)
 305                         print_hex_dump(KERN_INFO, "", DUMP_PREFIX_ADDRESS,
 306                                        16, 1, phys_to_virt(buffer_info->dma),
 307                                        buffer_info->length, true);
 308         }
 309
 310         /* Print Rx Ring Summary */
 311 rx_ring_summary:
 312         dev_info(&adapter->pdev->dev, "Rx Ring Summary\n");
 313         printk(KERN_INFO "Queue [NTU] [NTC]\n");
 314         printk(KERN_INFO " %5d %5X %5X\n", 0,
 315                rx_ring->next_to_use, rx_ring->next_to_clean);
 316
 317         /* Print Rx Ring */
 318         if (!netif_msg_rx_status(adapter))
 319                 goto exit;
 320
 321         dev_info(&adapter->pdev->dev, "Rx Ring Dump\n");
 322         switch (adapter->rx_ps_pages) {
 323         case 1:
 324         case 2:
 325         case 3:
 326                 /* [Extended] Packet Split Receive Descriptor Format
 327                  *
 328                  *    +-----------------------------------------------------+
 329                  *  0 |                Buffer Address 0 [63:0]              |
 330                  *    +-----------------------------------------------------+
 331                  *  8 |                Buffer Address 1 [63:0]              |
 332                  *    +-----------------------------------------------------+
 333                  * 16 |                Buffer Address 2 [63:0]              |
 334                  *    +-----------------------------------------------------+
 335                  * 24 |                Buffer Address 3 [63:0]              |
 336                  *    +-----------------------------------------------------+
 337                  */
 338                 printk(KERN_INFO "R  [desc]      [buffer 0 63:0 ] "
 339                        "[buffer 1 63:0 ] "
 340                        "[buffer 2 63:0 ] [buffer 3 63:0 ] [bi->dma       ] "
 341                        "[bi->skb] <-- Ext Pkt Split format\n");
 342                 /* [Extended] Receive Descriptor (Write-Back) Format
 343                  *
 344                  *   63       48 47    32 31     13 12    8 7    4 3        0
 345                  *   +------------------------------------------------------+
 346                  * 0 | Packet   | IP     |  Rsvd   | MRQ   | Rsvd | MRQ RSS |
 347                  *   | Checksum | Ident  |         | Queue |      |  Type   |
 348                  *   +------------------------------------------------------+
 349                  * 8 | VLAN Tag | Length | Extended Error | Extended Status |
 350                  *   +------------------------------------------------------+
 351                  *   63       48 47    32 31            20 19               0
 352                  */
 353                 printk(KERN_INFO "RWB[desc]      [ck ipid mrqhsh] "
 354                        "[vl   l0 ee  es] "
 355                        "[ l3  l2  l1 hs] [reserved      ] ---------------- "
 356                        "[bi->skb] <-- Ext Rx Write-Back format\n");
 357                 for (i = 0; i < rx_ring->count; i++) {
 358                         buffer_info = &rx_ring->buffer_info[i];
 359                         rx_desc_ps = E1000_RX_DESC_PS(*rx_ring, i);
 360                         u1 = (struct my_u1 *)rx_desc_ps;
 361                         staterr =
 362                             le32_to_cpu(rx_desc_ps->wb.middle.status_error);
 363                         if (staterr & E1000_RXD_STAT_DD) {
 364                                 /* Descriptor Done */
 365                                 printk(KERN_INFO "RWB[0x%03X]     %016llX "
 366                                        "%016llX %016llX %016llX "
 367                                        "---------------- %p", i,
 368                                        (unsigned long long)le64_to_cpu(u1->a),
 369                                        (unsigned long long)le64_to_cpu(u1->b),
 370                                        (unsigned long long)le64_to_cpu(u1->c),
 371                                        (unsigned long long)le64_to_cpu(u1->d),
 372                                        buffer_info->skb);
 373                         } else {
 374                                 printk(KERN_INFO "R  [0x%03X]     %016llX "
 375                                        "%016llX %016llX %016llX %016llX %p", i,
 376                                        (unsigned long long)le64_to_cpu(u1->a),
 377                                        (unsigned long long)le64_to_cpu(u1->b),
 378                                        (unsigned long long)le64_to_cpu(u1->c),
 379                                        (unsigned long long)le64_to_cpu(u1->d),
 380                                        (unsigned long long)buffer_info->dma,
 381                                        buffer_info->skb);
 382
 383                                 if (netif_msg_pktdata(adapter))
 384                                         print_hex_dump(KERN_INFO, "",
 385                                                 DUMP_PREFIX_ADDRESS, 16, 1,
 386                                                 phys_to_virt(buffer_info->dma),
 387                                                 adapter->rx_ps_bsize0, true);
 388                         }
 389
 390                         if (i == rx_ring->next_to_use)
 391                                 printk(KERN_CONT " NTU\n");
 392                         else if (i == rx_ring->next_to_clean)
 393                                 printk(KERN_CONT " NTC\n");
 394                         else
 395                                 printk(KERN_CONT "\n");
 396                 }
 397                 break;
 398         default:
 399         case 0:
 400                 /* Legacy Receive Descriptor Format
 401                  *
 402                  * +-----------------------------------------------------+
 403                  * |                Buffer Address [63:0]                |
 404                  * +-----------------------------------------------------+
 405                  * | VLAN Tag | Errors | Status 0 | Packet csum | Length |
 406                  * +-----------------------------------------------------+
 407                  * 63       48 47    40 39      32 31         16 15      0
 408                  */
 409                 printk(KERN_INFO "Rl[desc]     [address 63:0  ] "
 410                        "[vl er S cks ln] [bi->dma       ] [bi->skb] "
 411                        "<-- Legacy format\n");
 412                 for (i = 0; rx_ring->desc && (i < rx_ring->count); i++) {
 413                         rx_desc = E1000_RX_DESC(*rx_ring, i);
 414                         buffer_info = &rx_ring->buffer_info[i];
 415                         u0 = (struct my_u0 *)rx_desc;
 416                         printk(KERN_INFO "Rl[0x%03X]    %016llX %016llX "
 417                                "%016llX %p", i,
 418                                (unsigned long long)le64_to_cpu(u0->a),
 419                                (unsigned long long)le64_to_cpu(u0->b),
 420                                (unsigned long long)buffer_info->dma,
 421                                buffer_info->skb);
 422                         if (i == rx_ring->next_to_use)
 423                                 printk(KERN_CONT " NTU\n");
 424                         else if (i == rx_ring->next_to_clean)
 425                                 printk(KERN_CONT " NTC\n");
 426                         else
 427                                 printk(KERN_CONT "\n");
 428
 429                         if (netif_msg_pktdata(adapter))
 430                                 print_hex_dump(KERN_INFO, "",
 431                                                DUMP_PREFIX_ADDRESS,
 432                                                16, 1,
 433                                                phys_to_virt(buffer_info->dma),
 434                                                adapter->rx_buffer_len, true);
 435                 }
 436         }
 437
 438 exit:
 439         return;
 440 }
 441
 442 /**
 443  * e1000_desc_unused - calculate if we have unused descriptors
 444  **/
 445 static int e1000_desc_unused(struct e1000_ring *ring)
 446 {
 447         if (ring->next_to_clean > ring->next_to_use)
 448                 return ring->next_to_clean - ring->next_to_use - 1;
 449
 450         return ring->count + ring->next_to_clean - ring->next_to_use - 1;
 451 }
 452
 453 /**
 454  * e1000_receive_skb - helper function to handle Rx indications
 455  * @adapter: board private structure
 456  * @status: descriptor status field as written by hardware
 457  * @vlan: descriptor vlan field as written by hardware (no le/be conversion)
 458  * @skb: pointer to sk_buff to be indicated to stack
 459  **/
 460 static void e1000_receive_skb(struct e1000_adapter *adapter,
 461                               struct net_device *netdev, struct sk_buff *skb,
 462                               u8 status, __le16 vlan)
 463 {
 464         u16 tag = le16_to_cpu(vlan);
 465         skb->protocol = eth_type_trans(skb, netdev);
 466
 467         if (status & E1000_RXD_STAT_VP)
 468                 __vlan_hwaccel_put_tag(skb, tag);
 469
 470         napi_gro_receive(&adapter->napi, skb);
 471 }
 472
 473 /**
 474  * e1000_rx_checksum - Receive Checksum Offload
 475  * @adapter:     board private structure
 476  * @status_err:  receive descriptor status and error fields
 477  * @csum:       receive descriptor csum field
 478  * @sk_buff:     socket buffer with received data
 479  **/
 480 static void e1000_rx_checksum(struct e1000_adapter *adapter, u32 status_err,
 481                               u32 csum, struct sk_buff *skb)
 482 {
 483         u16 status = (u16)status_err;
 484         u8 errors = (u8)(status_err >> 24);
 485
 486         skb_checksum_none_assert(skb);
 487
 488         /* Ignore Checksum bit is set */
 489         if (status & E1000_RXD_STAT_IXSM)
 490                 return;
 491         /* TCP/UDP checksum error bit is set */
 492         if (errors & E1000_RXD_ERR_TCPE) {
 493                 /* let the stack verify checksum errors */
 494                 adapter->hw_csum_err++;
 495                 return;
 496         }
 497
 498         /* TCP/UDP Checksum has not been calculated */
 499         if (!(status & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS)))
 500                 return;
 501
 502         /* It must be a TCP or UDP packet with a valid checksum */
 503         if (status & E1000_RXD_STAT_TCPCS) {
 504                 /* TCP checksum is good */
 505                 skb->ip_summed = CHECKSUM_UNNECESSARY;
 506         } else {
 507                 /*
 508                  * IP fragment with UDP payload
 509                  * Hardware complements the payload checksum, so we undo it
 510                  * and then put the value in host order for further stack use.
 511                  */
 512                 __sum16 sum = (__force __sum16)htons(csum);
 513                 skb->csum = csum_unfold(~sum);
 514                 skb->ip_summed = CHECKSUM_COMPLETE;
 515         }
 516         adapter->hw_csum_good++;
 517 }
 518
 519 /**
 520  * e1000_alloc_rx_buffers - Replace used receive buffers; legacy & extended
 521  * @adapter: address of board private structure
 522  **/
 523 static void e1000_alloc_rx_buffers(struct e1000_adapter *adapter,
 524                                    int cleaned_count)
 525 {
 526         struct net_device *netdev = adapter->netdev;
 527         struct pci_dev *pdev = adapter->pdev;
 528         struct e1000_ring *rx_ring = adapter->rx_ring;
 529         struct e1000_rx_desc *rx_desc;
 530         struct e1000_buffer *buffer_info;
 531         struct sk_buff *skb;
 532         unsigned int i;
 533         unsigned int bufsz = adapter->rx_buffer_len;
 534
 535         i = rx_ring->next_to_use;
 536         buffer_info = &rx_ring->buffer_info[i];
 537
 538         while (cleaned_count--) {
 539                 skb = buffer_info->skb;
 540                 if (skb) {
 541                         skb_trim(skb, 0);
 542                         goto map_skb;
 543                 }
 544
 545                 skb = netdev_alloc_skb_ip_align(netdev, bufsz);
 546                 if (!skb) {
 547                         /* Better luck next round */
 548                         adapter->alloc_rx_buff_failed++;
 549                         break;
 550                 }
 551
 552                 buffer_info->skb = skb;
 553 map_skb:
 554                 buffer_info->dma = dma_map_single(&pdev->dev, skb->data,
 555                                                   adapter->rx_buffer_len,
 556                                                   DMA_FROM_DEVICE);
 557                 if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {
 558                         dev_err(&pdev->dev, "Rx DMA map failed\n");
 559                         adapter->rx_dma_failed++;
 560                         break;
 561                 }
 562
 563                 rx_desc = E1000_RX_DESC(*rx_ring, i);
 564                 rx_desc->buffer_addr = cpu_to_le64(buffer_info->dma);
 565
 566                 if (unlikely(!(i & (E1000_RX_BUFFER_WRITE - 1)))) {
 567                         /*
 568                          * Force memory writes to complete before letting h/w
 569                          * know there are new descriptors to fetch.  (Only
 570                          * applicable for weak-ordered memory model archs,
 571                          * such as IA-64).
 572                          */
 573                         wmb();
 574                         writel(i, adapter->hw.hw_addr + rx_ring->tail);
 575                 }
 576                 i++;
 577                 if (i == rx_ring->count)
 578                         i = 0;
 579                 buffer_info = &rx_ring->buffer_info[i];
 580         }
 581
 582         rx_ring->next_to_use = i;
 583 }
 584
 585 /**
 586  * e1000_alloc_rx_buffers_ps - Replace used receive buffers; packet split
 587  * @adapter: address of board private structure
 588  **/
 589 static void e1000_alloc_rx_buffers_ps(struct e1000_adapter *adapter,
 590                                       int cleaned_count)
 591 {
 592         struct net_device *netdev = adapter->netdev;
 593         struct pci_dev *pdev = adapter->pdev;
 594         union e1000_rx_desc_packet_split *rx_desc;
 595         struct e1000_ring *rx_ring = adapter->rx_ring;
 596         struct e1000_buffer *buffer_info;
 597         struct e1000_ps_page *ps_page;
 598         struct sk_buff *skb;
 599         unsigned int i, j;
 600
 601         i = rx_ring->next_to_use;
 602         buffer_info = &rx_ring->buffer_info[i];
 603
 604         while (cleaned_count--) {
 605                 rx_desc = E1000_RX_DESC_PS(*rx_ring, i);
 606
 607                 for (j = 0; j < PS_PAGE_BUFFERS; j++) {
 608                         ps_page = &buffer_info->ps_pages[j];
 609                         if (j >= adapter->rx_ps_pages) {
 610                                 /* all unused desc entries get hw null ptr */
 611                                 rx_desc->read.buffer_addr[j + 1] =
 612                                     ~cpu_to_le64(0);
 613                                 continue;
 614                         }
 615                         if (!ps_page->page) {
 616                                 ps_page->page = alloc_page(GFP_ATOMIC);
 617                                 if (!ps_page->page) {
 618                                         adapter->alloc_rx_buff_failed++;
 619                                         goto no_buffers;
 620                                 }
 621                                 ps_page->dma = dma_map_page(&pdev->dev,
 622                                                             ps_page->page,
 623                                                             0, PAGE_SIZE,
 624                                                             DMA_FROM_DEVICE);
 625                                 if (dma_mapping_error(&pdev->dev,
 626                                                       ps_page->dma)) {
 627                                         dev_err(&adapter->pdev->dev,
 628                                                 "Rx DMA page map failed\n");
 629                                         adapter->rx_dma_failed++;
 630                                         goto no_buffers;
 631                                 }
 632                         }
 633                         /*
 634                          * Refresh the desc even if buffer_addrs
 635                          * didn't change because each write-back
 636                          * erases this info.
 637                          */
 638                         rx_desc->read.buffer_addr[j + 1] =
 639                             cpu_to_le64(ps_page->dma);
 640                 }
 641
 642                 skb = netdev_alloc_skb_ip_align(netdev,
 643                                                 adapter->rx_ps_bsize0);
 644
 645                 if (!skb) {
 646                         adapter->alloc_rx_buff_failed++;
 647                         break;
 648                 }
 649
 650                 buffer_info->skb = skb;
 651                 buffer_info->dma = dma_map_single(&pdev->dev, skb->data,
 652                                                   adapter->rx_ps_bsize0,
 653                                                   DMA_FROM_DEVICE);
 654                 if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {
 655                         dev_err(&pdev->dev, "Rx DMA map failed\n");
 656                         adapter->rx_dma_failed++;
 657                         /* cleanup skb */
 658                         dev_kfree_skb_any(skb);
 659                         buffer_info->skb = NULL;
 660                         break;
 661                 }
 662
 663                 rx_desc->read.buffer_addr[0] = cpu_to_le64(buffer_info->dma);
 664
 665                 if (unlikely(!(i & (E1000_RX_BUFFER_WRITE - 1)))) {
 666                         /*
 667                          * Force memory writes to complete before letting h/w
 668                          * know there are new descriptors to fetch.  (Only
 669                          * applicable for weak-ordered memory model archs,
 670                          * such as IA-64).
 671                          */
 672                         wmb();
 673                         writel(i << 1, adapter->hw.hw_addr + rx_ring->tail);
 674                 }
 675
 676                 i++;
 677                 if (i == rx_ring->count)
 678                         i = 0;
 679                 buffer_info = &rx_ring->buffer_info[i];
 680         }
 681
 682 no_buffers:
 683         rx_ring->next_to_use = i;
 684 }
 685
 686 /**
 687  * e1000_alloc_jumbo_rx_buffers - Replace used jumbo receive buffers
 688  * @adapter: address of board private structure
 689  * @cleaned_count: number of buffers to allocate this pass
 690  **/
 691
 692 static void e1000_alloc_jumbo_rx_buffers(struct e1000_adapter *adapter,
 693                                          int cleaned_count)
 694 {
 695         struct net_device *netdev = adapter->netdev;
 696         struct pci_dev *pdev = adapter->pdev;
 697         struct e1000_rx_desc *rx_desc;
 698         struct e1000_ring *rx_ring = adapter->rx_ring;
 699         struct e1000_buffer *buffer_info;
 700         struct sk_buff *skb;
 701         unsigned int i;
 702         unsigned int bufsz = 256 - 16 /* for skb_reserve */;
 703
 704         i = rx_ring->next_to_use;
 705         buffer_info = &rx_ring->buffer_info[i];
 706
 707         while (cleaned_count--) {
 708                 skb = buffer_info->skb;
 709                 if (skb) {
 710                         skb_trim(skb, 0);
 711                         goto check_page;
 712                 }
 713
 714                 skb = netdev_alloc_skb_ip_align(netdev, bufsz);
 715                 if (unlikely(!skb)) {
 716                         /* Better luck next round */
 717                         adapter->alloc_rx_buff_failed++;
 718                         break;
 719                 }
 720
 721                 buffer_info->skb = skb;
 722 check_page:
 723                 /* allocate a new page if necessary */
 724                 if (!buffer_info->page) {
 725                         buffer_info->page = alloc_page(GFP_ATOMIC);
 726                         if (unlikely(!buffer_info->page)) {
 727                                 adapter->alloc_rx_buff_failed++;
 728                                 break;
 729                         }
 730                 }
 731
 732                 if (!buffer_info->dma)
 733                         buffer_info->dma = dma_map_page(&pdev->dev,
 734                                                         buffer_info->page, 0,
 735                                                         PAGE_SIZE,
 736                                                         DMA_FROM_DEVICE);
 737
 738                 rx_desc = E1000_RX_DESC(*rx_ring, i);
 739                 rx_desc->buffer_addr = cpu_to_le64(buffer_info->dma);
 740
 741                 if (unlikely(++i == rx_ring->count))
 742                         i = 0;
 743                 buffer_info = &rx_ring->buffer_info[i];
 744         }
 745
 746         if (likely(rx_ring->next_to_use != i)) {
 747                 rx_ring->next_to_use = i;
 748                 if (unlikely(i-- == 0))
 749                         i = (rx_ring->count - 1);
 750
 751                 /* Force memory writes to complete before letting h/w
 752                  * know there are new descriptors to fetch.  (Only
 753                  * applicable for weak-ordered memory model archs,
 754                  * such as IA-64). */
 755                 wmb();
 756                 writel(i, adapter->hw.hw_addr + rx_ring->tail);
 757         }
 758 }
 759
 760 /**
 761  * e1000_clean_rx_irq - Send received data up the network stack; legacy
 762  * @adapter: board private structure
 763  *
 764  * the return value indicates whether actual cleaning was done, there
 765  * is no guarantee that everything was cleaned
 766  **/
 767 static bool e1000_clean_rx_irq(struct e1000_adapter *adapter,
 768                                int *work_done, int work_to_do)
 769 {
 770         struct net_device *netdev = adapter->netdev;
 771         struct pci_dev *pdev = adapter->pdev;
 772         struct e1000_hw *hw = &adapter->hw;
 773         struct e1000_ring *rx_ring = adapter->rx_ring;
 774         struct e1000_rx_desc *rx_desc, *next_rxd;
 775         struct e1000_buffer *buffer_info, *next_buffer;
 776         u32 length;
 777         unsigned int i;
 778         int cleaned_count = 0;
 779         bool cleaned = 0;
 780         unsigned int total_rx_bytes = 0, total_rx_packets = 0;
 781
 782         i = rx_ring->next_to_clean;
 783         rx_desc = E1000_RX_DESC(*rx_ring, i);
 784         buffer_info = &rx_ring->buffer_info[i];
 785
 786         while (rx_desc->status & E1000_RXD_STAT_DD) {
 787                 struct sk_buff *skb;
 788                 u8 status;
 789
 790                 if (*work_done >= work_to_do)
 791                         break;
 792                 (*work_done)++;
 793                 rmb();  /* read descriptor and rx_buffer_info after status DD */
 794
 795                 status = rx_desc->status;
 796                 skb = buffer_info->skb;
 797                 buffer_info->skb = NULL;
 798
 799                 prefetch(skb->data - NET_IP_ALIGN);
 800
 801                 i++;
 802                 if (i == rx_ring->count)
 803                         i = 0;
 804                 next_rxd = E1000_RX_DESC(*rx_ring, i);
 805                 prefetch(next_rxd);
 806
 807                 next_buffer = &rx_ring->buffer_info[i];
 808
 809                 cleaned = 1;
 810                 cleaned_count++;
 811                 dma_unmap_single(&pdev->dev,
 812                                  buffer_info->dma,
 813                                  adapter->rx_buffer_len,
 814                                  DMA_FROM_DEVICE);
 815                 buffer_info->dma = 0;
 816
 817                 length = le16_to_cpu(rx_desc->length);
 818
 819                 /*
 820                  * !EOP means multiple descriptors were used to store a single
 821                  * packet, if that's the case we need to toss it.  In fact, we
 822                  * need to toss every packet with the EOP bit clear and the
 823                  * next frame that _does_ have the EOP bit set, as it is by
 824                  * definition only a frame fragment
 825                  */
 826                 if (unlikely(!(status & E1000_RXD_STAT_EOP)))
 827                         adapter->flags2 |= FLAG2_IS_DISCARDING;
 828
 829                 if (adapter->flags2 & FLAG2_IS_DISCARDING) {
 830                         /* All receives must fit into a single buffer */
 831                         e_dbg("Receive packet consumed multiple buffers\n");
 832                         /* recycle */
 833                         buffer_info->skb = skb;
 834                         if (status & E1000_RXD_STAT_EOP)
 835                                 adapter->flags2 &= ~FLAG2_IS_DISCARDING;
 836                         goto next_desc;
 837                 }
 838
 839                 if (rx_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK) {
 840                         /* recycle */
 841                         buffer_info->skb = skb;
 842                         goto next_desc;
 843                 }
 844
 845                 /* adjust length to remove Ethernet CRC */
 846                 if (!(adapter->flags2 & FLAG2_CRC_STRIPPING))
 847                         length -= 4;
 848
 849                 total_rx_bytes += length;
 850                 total_rx_packets++;
 851
 852                 /*
 853                  * code added for copybreak, this should improve
 854                  * performance for small packets with large amounts
 855                  * of reassembly being done in the stack
 856                  */
 857                 if (length < copybreak) {
 858                         struct sk_buff *new_skb =
 859                             netdev_alloc_skb_ip_align(netdev, length);
 860                         if (new_skb) {
 861                                 skb_copy_to_linear_data_offset(new_skb,
 862                                                                -NET_IP_ALIGN,
 863                                                                (skb->data -
 864                                                                 NET_IP_ALIGN),
 865                                                                (length +
 866                                                                 NET_IP_ALIGN));
 867                                 /* save the skb in buffer_info as good */
 868                                 buffer_info->skb = skb;
 869                                 skb = new_skb;
 870                         }
 871                         /* else just continue with the old one */
 872                 }
 873                 /* end copybreak code */
 874                 skb_put(skb, length);
 875
 876                 /* Receive Checksum Offload */
 877                 e1000_rx_checksum(adapter,
 878                                   (u32)(status) |
 879                                   ((u32)(rx_desc->errors) << 24),
 880                                   le16_to_cpu(rx_desc->csum), skb);
 881
 882                 e1000_receive_skb(adapter, netdev, skb,status,rx_desc->special);
 883
 884 next_desc:
 885                 rx_desc->status = 0;
 886
 887                 /* return some buffers to hardware, one at a time is too slow */
 888                 if (cleaned_count >= E1000_RX_BUFFER_WRITE) {
 889                         adapter->alloc_rx_buf(adapter, cleaned_count);
 890                         cleaned_count = 0;
 891                 }
 892
 893                 /* use prefetched values */
 894                 rx_desc = next_rxd;
 895                 buffer_info = next_buffer;
 896         }
 897         rx_ring->next_to_clean = i;
 898
 899         cleaned_count = e1000_desc_unused(rx_ring);
 900         if (cleaned_count)
 901                 adapter->alloc_rx_buf(adapter, cleaned_count);
 902
 903         adapter->total_rx_bytes += total_rx_bytes;
 904         adapter->total_rx_packets += total_rx_packets;
 905         return cleaned;
 906 }
 907
 908 static void e1000_put_txbuf(struct e1000_adapter *adapter,
 909                              struct e1000_buffer *buffer_info)
 910 {
 911         if (buffer_info->dma) {
 912                 if (buffer_info->mapped_as_page)
 913                         dma_unmap_page(&adapter->pdev->dev, buffer_info->dma,
 914                                        buffer_info->length, DMA_TO_DEVICE);
 915                 else
 916                         dma_unmap_single(&adapter->pdev->dev, buffer_info->dma,
 917                                          buffer_info->length, DMA_TO_DEVICE);
 918                 buffer_info->dma = 0;
 919         }
 920         if (buffer_info->skb) {
 921                 dev_kfree_skb_any(buffer_info->skb);
 922                 buffer_info->skb = NULL;
 923         }
 924         buffer_info->time_stamp = 0;
 925 }
 926
 927 static void e1000_print_hw_hang(struct work_struct *work)
 928 {
 929         struct e1000_adapter *adapter = container_of(work,
 930                                                      struct e1000_adapter,
 931                                                      print_hang_task);
 932         struct e1000_ring *tx_ring = adapter->tx_ring;
 933         unsigned int i = tx_ring->next_to_clean;
 934         unsigned int eop = tx_ring->buffer_info[i].next_to_watch;
 935         struct e1000_tx_desc *eop_desc = E1000_TX_DESC(*tx_ring, eop);
 936         struct e1000_hw *hw = &adapter->hw;
 937         u16 phy_status, phy_1000t_status, phy_ext_status;
 938         u16 pci_status;
 939
 940         if (test_bit(__E1000_DOWN, &adapter->state))
 941                 return;
 942
 943         e1e_rphy(hw, PHY_STATUS, &phy_status);
 944         e1e_rphy(hw, PHY_1000T_STATUS, &phy_1000t_status);
 945         e1e_rphy(hw, PHY_EXT_STATUS, &phy_ext_status);
 946
 947         pci_read_config_word(adapter->pdev, PCI_STATUS, &pci_status);
 948
 949         /* detected Hardware unit hang */
 950         e_err("Detected Hardware Unit Hang:\n"
 951               "  TDH                  <%x>\n"
 952               "  TDT                  <%x>\n"
 953               "  next_to_use          <%x>\n"
 954               "  next_to_clean        <%x>\n"
 955               "buffer_info[next_to_clean]:\n"
 956               "  time_stamp           <%lx>\n"
 957               "  next_to_watch        <%x>\n"
 958               "  jiffies              <%lx>\n"
 959               "  next_to_watch.status <%x>\n"
 960               "MAC Status             <%x>\n"
 961               "PHY Status             <%x>\n"
 962               "PHY 1000BASE-T Status  <%x>\n"
 963               "PHY Extended Status    <%x>\n"
 964               "PCI Status             <%x>\n",
 965               readl(adapter->hw.hw_addr + tx_ring->head),
 966               readl(adapter->hw.hw_addr + tx_ring->tail),
 967               tx_ring->next_to_use,
 968               tx_ring->next_to_clean,
 969               tx_ring->buffer_info[eop].time_stamp,
 970               eop,
 971               jiffies,
 972               eop_desc->upper.fields.status,
 973               er32(STATUS),
 974               phy_status,
 975               phy_1000t_status,
 976               phy_ext_status,
 977               pci_status);
 978 }
 979
 980 /**
 981  * e1000_clean_tx_irq - Reclaim resources after transmit completes
 982  * @adapter: board private structure
 983  *
 984  * the return value indicates whether actual cleaning was done, there
 985  * is no guarantee that everything was cleaned
 986  **/
 987 static bool e1000_clean_tx_irq(struct e1000_adapter *adapter)
 988 {
 989         struct net_device *netdev = adapter->netdev;
 990         struct e1000_hw *hw = &adapter->hw;
 991         struct e1000_ring *tx_ring = adapter->tx_ring;
 992         struct e1000_tx_desc *tx_desc, *eop_desc;
 993         struct e1000_buffer *buffer_info;
 994         unsigned int i, eop;
 995         unsigned int count = 0;
 996         unsigned int total_tx_bytes = 0, total_tx_packets = 0;
 997
 998         i = tx_ring->next_to_clean;
 999         eop = tx_ring->buffer_info[i].next_to_watch;
1000         eop_desc = E1000_TX_DESC(*tx_ring, eop);
1001
1002         while ((eop_desc->upper.data & cpu_to_le32(E1000_TXD_STAT_DD)) &&
1003                (count < tx_ring->count)) {
1004                 bool cleaned = false;
1005                 rmb(); /* read buffer_info after eop_desc */
1006                 for (; !cleaned; count++) {
1007                         tx_desc = E1000_TX_DESC(*tx_ring, i);
1008                         buffer_info = &tx_ring->buffer_info[i];
1009                         cleaned = (i == eop);
1010
1011                         if (cleaned) {
1012                                 total_tx_packets += buffer_info->segs;
1013                                 total_tx_bytes += buffer_info->bytecount;
1014                         }
1015
1016                         e1000_put_txbuf(adapter, buffer_info);
1017                         tx_desc->upper.data = 0;
1018
1019                         i++;
1020                         if (i == tx_ring->count)
1021                                 i = 0;
1022                 }
1023
1024                 if (i == tx_ring->next_to_use)
1025                         break;
1026                 eop = tx_ring->buffer_info[i].next_to_watch;
1027                 eop_desc = E1000_TX_DESC(*tx_ring, eop);
1028         }
1029
1030         tx_ring->next_to_clean = i;
1031
1032 #define TX_WAKE_THRESHOLD 32
1033         if (count && netif_carrier_ok(netdev) &&
1034             e1000_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD) {
1035                 /* Make sure that anybody stopping the queue after this
1036                  * sees the new next_to_clean.
1037                  */
1038                 smp_mb();
1039
1040                 if (netif_queue_stopped(netdev) &&
1041                     !(test_bit(__E1000_DOWN, &adapter->state))) {
1042                         netif_wake_queue(netdev);
1043                         ++adapter->restart_queue;
1044                 }
1045         }
1046
1047         if (adapter->detect_tx_hung) {
1048                 /*
1049                  * Detect a transmit hang in hardware, this serializes the
1050                  * check with the clearing of time_stamp and movement of i
1051                  */
1052                 adapter->detect_tx_hung = 0;
1053                 if (tx_ring->buffer_info[i].time_stamp &&
1054                     time_after(jiffies, tx_ring->buffer_info[i].time_stamp
1055                                + (adapter->tx_timeout_factor * HZ)) &&
1056                     !(er32(STATUS) & E1000_STATUS_TXOFF)) {
1057                         schedule_work(&adapter->print_hang_task);
1058                         netif_stop_queue(netdev);
1059                 }
1060         }
1061         adapter->total_tx_bytes += total_tx_bytes;
1062         adapter->total_tx_packets += total_tx_packets;
1063         return count < tx_ring->count;
1064 }
1065
1066 /**
1067  * e1000_clean_rx_irq_ps - Send received data up the network stack; packet split
1068  * @adapter: board private structure
1069  *
1070  * the return value indicates whether actual cleaning was done, there
1071  * is no guarantee that everything was cleaned
1072  **/
1073 static bool e1000_clean_rx_irq_ps(struct e1000_adapter *adapter,
1074                                   int *work_done, int work_to_do)
1075 {
1076         struct e1000_hw *hw = &adapter->hw;
1077         union e1000_rx_desc_packet_split *rx_desc, *next_rxd;
1078         struct net_device *netdev = adapter->netdev;
1079         struct pci_dev *pdev = adapter->pdev;
1080         struct e1000_ring *rx_ring = adapter->rx_ring;
1081         struct e1000_buffer *buffer_info, *next_buffer;
1082         struct e1000_ps_page *ps_page;
1083         struct sk_buff *skb;
1084         unsigned int i, j;
1085         u32 length, staterr;
1086         int cleaned_count = 0;
1087         bool cleaned = 0;
1088         unsigned int total_rx_bytes = 0, total_rx_packets = 0;
1089
1090         i = rx_ring->next_to_clean;
1091         rx_desc = E1000_RX_DESC_PS(*rx_ring, i);
1092         staterr = le32_to_cpu(rx_desc->wb.middle.status_error);
1093         buffer_info = &rx_ring->buffer_info[i];
1094
1095         while (staterr & E1000_RXD_STAT_DD) {
1096                 if (*work_done >= work_to_do)
1097                         break;
1098                 (*work_done)++;
1099                 skb = buffer_info->skb;
1100                 rmb();  /* read descriptor and rx_buffer_info after status DD */
1101
1102                 /* in the packet split case this is header only */
1103                 prefetch(skb->data - NET_IP_ALIGN);
1104
1105                 i++;
1106                 if (i == rx_ring->count)
1107                         i = 0;
1108                 next_rxd = E1000_RX_DESC_PS(*rx_ring, i);
1109                 prefetch(next_rxd);
1110
1111                 next_buffer = &rx_ring->buffer_info[i];
1112
1113                 cleaned = 1;
1114                 cleaned_count++;
1115                 dma_unmap_single(&pdev->dev, buffer_info->dma,
1116                                  adapter->rx_ps_bsize0, DMA_FROM_DEVICE);
1117                 buffer_info->dma = 0;
1118
1119                 /* see !EOP comment in other Rx routine */
1120                 if (!(staterr & E1000_RXD_STAT_EOP))
1121                         adapter->flags2 |= FLAG2_IS_DISCARDING;
1122
1123                 if (adapter->flags2 & FLAG2_IS_DISCARDING) {
1124                         e_dbg("Packet Split buffers didn't pick up the full "
1125                               "packet\n");
1126                         dev_kfree_skb_irq(skb);
1127                         if (staterr & E1000_RXD_STAT_EOP)
1128                                 adapter->flags2 &= ~FLAG2_IS_DISCARDING;
1129                         goto next_desc;
1130                 }
1131
1132                 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
1133                         dev_kfree_skb_irq(skb);
1134                         goto next_desc;
1135                 }
1136
1137                 length = le16_to_cpu(rx_desc->wb.middle.length0);
1138
1139                 if (!length) {
1140                         e_dbg("Last part of the packet spanning multiple "
1141                               "descriptors\n");
1142                         dev_kfree_skb_irq(skb);
1143                         goto next_desc;
1144                 }
1145
1146                 /* Good Receive */
1147                 skb_put(skb, length);
1148
1149                 {
1150                 /*
1151                  * this looks ugly, but it seems compiler issues make it
1152                  * more efficient than reusing j
1153                  */
1154                 int l1 = le16_to_cpu(rx_desc->wb.upper.length[0]);
1155
1156                 /*
1157                  * page alloc/put takes too long and effects small packet
1158                  * throughput, so unsplit small packets and save the alloc/put
1159                  * only valid in softirq (napi) context to call kmap_*
1160                  */
1161                 if (l1 && (l1 <= copybreak) &&
1162                     ((length + l1) <= adapter->rx_ps_bsize0)) {
1163                         u8 *vaddr;
1164
1165                         ps_page = &buffer_info->ps_pages[0];
1166
1167                         /*
1168                          * there is no documentation about how to call
1169                          * kmap_atomic, so we can't hold the mapping
1170                          * very long
1171                          */
1172                         dma_sync_single_for_cpu(&pdev->dev, ps_page->dma,
1173                                                 PAGE_SIZE, DMA_FROM_DEVICE);
1174                         vaddr = kmap_atomic(ps_page->page, KM_SKB_DATA_SOFTIRQ);
1175                         memcpy(skb_tail_pointer(skb), vaddr, l1);
1176                         kunmap_atomic(vaddr, KM_SKB_DATA_SOFTIRQ);
1177                         dma_sync_single_for_device(&pdev->dev, ps_page->dma,
1178                                                    PAGE_SIZE, DMA_FROM_DEVICE);
1179
1180                         /* remove the CRC */
1181                         if (!(adapter->flags2 & FLAG2_CRC_STRIPPING))
1182                                 l1 -= 4;
1183
1184                         skb_put(skb, l1);
1185                         goto copydone;
1186                 } /* if */
1187                 }
1188
1189                 for (j = 0; j < PS_PAGE_BUFFERS; j++) {
1190                         length = le16_to_cpu(rx_desc->wb.upper.length[j]);
1191                         if (!length)
1192                                 break;
1193
1194                         ps_page = &buffer_info->ps_pages[j];
1195                         dma_unmap_page(&pdev->dev, ps_page->dma, PAGE_SIZE,
1196                                        DMA_FROM_DEVICE);
1197                         ps_page->dma = 0;
1198                         skb_fill_page_desc(skb, j, ps_page->page, 0, length);
1199                         ps_page->page = NULL;
1200                         skb->len += length;
1201                         skb->data_len += length;
1202                         skb->truesize += length;
1203                 }
1204
1205                 /* strip the ethernet crc, problem is we're using pages now so
1206                  * this whole operation can get a little cpu intensive
1207                  */
1208                 if (!(adapter->flags2 & FLAG2_CRC_STRIPPING))
1209                         pskb_trim(skb, skb->len - 4);
1210
1211 copydone:
1212                 total_rx_bytes += skb->len;
1213                 total_rx_packets++;
1214
1215                 e1000_rx_checksum(adapter, staterr, le16_to_cpu(
1216                         rx_desc->wb.lower.hi_dword.csum_ip.csum), skb);
1217
1218                 if (rx_desc->wb.upper.header_status &
1219                            cpu_to_le16(E1000_RXDPS_HDRSTAT_HDRSP))
1220                         adapter->rx_hdr_split++;
1221
1222                 e1000_receive_skb(adapter, netdev, skb,
1223                                   staterr, rx_desc->wb.middle.vlan);
1224
1225 next_desc:
1226                 rx_desc->wb.middle.status_error &= cpu_to_le32(~0xFF);
1227                 buffer_info->skb = NULL;
1228
1229                 /* return some buffers to hardware, one at a time is too slow */
1230                 if (cleaned_count >= E1000_RX_BUFFER_WRITE) {
1231                         adapter->alloc_rx_buf(adapter, cleaned_count);
1232                         cleaned_count = 0;
1233                 }
1234
1235                 /* use prefetched values */
1236                 rx_desc = next_rxd;
1237                 buffer_info = next_buffer;
1238
1239                 staterr = le32_to_cpu(rx_desc->wb.middle.status_error);
1240         }
1241         rx_ring->next_to_clean = i;
1242
1243         cleaned_count = e1000_desc_unused(rx_ring);
1244         if (cleaned_count)
1245                 adapter->alloc_rx_buf(adapter, cleaned_count);
1246
1247         adapter->total_rx_bytes += total_rx_bytes;
1248         adapter->total_rx_packets += total_rx_packets;
1249         return cleaned;
1250 }
1251
1252 /**
1253  * e1000_consume_page - helper function
1254  **/
1255 static void e1000_consume_page(struct e1000_buffer *bi, struct sk_buff *skb,
1256                                u16 length)
1257 {
1258         bi->page = NULL;
1259         skb->len += length;
1260         skb->data_len += length;
1261         skb->truesize += length;
1262 }
1263
1264 /**
1265  * e1000_clean_jumbo_rx_irq - Send received data up the network stack; legacy
1266  * @adapter: board private structure
1267  *
1268  * the return value indicates whether actual cleaning was done, there
1269  * is no guarantee that everything was cleaned
1270  **/
1271
1272 static bool e1000_clean_jumbo_rx_irq(struct e1000_adapter *adapter,
1273                                      int *work_done, int work_to_do)
1274 {
1275         struct net_device *netdev = adapter->netdev;
1276         struct pci_dev *pdev = adapter->pdev;
1277         struct e1000_ring *rx_ring = adapter->rx_ring;
1278         struct e1000_rx_desc *rx_desc, *next_rxd;
1279         struct e1000_buffer *buffer_info, *next_buffer;
1280         u32 length;
1281         unsigned int i;
1282         int cleaned_count = 0;
1283         bool cleaned = false;
1284         unsigned int total_rx_bytes=0, total_rx_packets=0;
1285
1286         i = rx_ring->next_to_clean;
1287         rx_desc = E1000_RX_DESC(*rx_ring, i);
1288         buffer_info = &rx_ring->buffer_info[i];
1289
1290         while (rx_desc->status & E1000_RXD_STAT_DD) {
1291                 struct sk_buff *skb;
1292                 u8 status;
1293
1294                 if (*work_done >= work_to_do)
1295                         break;
1296                 (*work_done)++;
1297                 rmb();  /* read descriptor and rx_buffer_info after status DD */
1298
1299                 status = rx_desc->status;
1300                 skb = buffer_info->skb;
1301                 buffer_info->skb = NULL;
1302
1303                 ++i;
1304                 if (i == rx_ring->count)
1305                         i = 0;
1306                 next_rxd = E1000_RX_DESC(*rx_ring, i);
1307                 prefetch(next_rxd);
1308
1309                 next_buffer = &rx_ring->buffer_info[i];
1310
1311                 cleaned = true;
1312                 cleaned_count++;
1313                 dma_unmap_page(&pdev->dev, buffer_info->dma, PAGE_SIZE,
1314                                DMA_FROM_DEVICE);
1315                 buffer_info->dma = 0;
1316
1317                 length = le16_to_cpu(rx_desc->length);
1318
1319                 /* errors is only valid for DD + EOP descriptors */
1320                 if (unlikely((status & E1000_RXD_STAT_EOP) &&
1321                     (rx_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK))) {
1322                                 /* recycle both page and skb */
1323                                 buffer_info->skb = skb;
1324                                 /* an error means any chain goes out the window
1325                                  * too */
1326                                 if (rx_ring->rx_skb_top)
1327                                         dev_kfree_skb_irq(rx_ring->rx_skb_top);
1328                                 rx_ring->rx_skb_top = NULL;
1329                                 goto next_desc;
1330                 }
1331
1332 #define rxtop (rx_ring->rx_skb_top)
1333                 if (!(status & E1000_RXD_STAT_EOP)) {
1334                         /* this descriptor is only the beginning (or middle) */
1335                         if (!rxtop) {
1336                                 /* this is the beginning of a chain */
1337                                 rxtop = skb;
1338                                 skb_fill_page_desc(rxtop, 0, buffer_info->page,
1339                                                    0, length);
1340                         } else {
1341                                 /* this is the middle of a chain */
1342                                 skb_fill_page_desc(rxtop,
1343                                     skb_shinfo(rxtop)->nr_frags,
1344                                     buffer_info->page, 0, length);
1345                                 /* re-use the skb, only consumed the page */
1346                                 buffer_info->skb = skb;
1347                         }
1348                         e1000_consume_page(buffer_info, rxtop, length);
1349                         goto next_desc;
1350                 } else {
1351                         if (rxtop) {
1352                                 /* end of the chain */
1353                                 skb_fill_page_desc(rxtop,
1354                                     skb_shinfo(rxtop)->nr_frags,
1355                                     buffer_info->page, 0, length);
1356                                 /* re-use the current skb, we only consumed the
1357                                  * page */
1358                                 buffer_info->skb = skb;
1359                                 skb = rxtop;
1360                                 rxtop = NULL;
1361                                 e1000_consume_page(buffer_info, skb, length);
1362                         } else {
1363                                 /* no chain, got EOP, this buf is the packet
1364                                  * copybreak to save the put_page/alloc_page */
1365                                 if (length <= copybreak &&
1366                                     skb_tailroom(skb) >= length) {
1367                                         u8 *vaddr;
1368                                         vaddr = kmap_atomic(buffer_info->page,
1369                                                            KM_SKB_DATA_SOFTIRQ);
1370                                         memcpy(skb_tail_pointer(skb), vaddr,
1371                                                length);
1372                                         kunmap_atomic(vaddr,
1373                                                       KM_SKB_DATA_SOFTIRQ);
1374                                         /* re-use the page, so don't erase
1375                                          * buffer_info->page */
1376                                         skb_put(skb, length);
1377                                 } else {
1378                                         skb_fill_page_desc(skb, 0,
1379                                                            buffer_info->page, 0,
1380                                                            length);
1381                                         e1000_consume_page(buffer_info, skb,
1382                                                            length);
1383                                 }
1384                         }
1385                 }
1386
1387                 /* Receive Checksum Offload XXX recompute due to CRC strip? */
1388                 e1000_rx_checksum(adapter,
1389                                   (u32)(status) |
1390                                   ((u32)(rx_desc->errors) << 24),
1391                                   le16_to_cpu(rx_desc->csum), skb);
1392
1393                 /* probably a little skewed due to removing CRC */
1394                 total_rx_bytes += skb->len;
1395                 total_rx_packets++;
1396
1397                 /* eth type trans needs skb->data to point to something */
1398                 if (!pskb_may_pull(skb, ETH_HLEN)) {
1399                         e_err("pskb_may_pull failed.\n");
1400                         dev_kfree_skb_irq(skb);
1401                         goto next_desc;
1402                 }
1403
1404                 e1000_receive_skb(adapter, netdev, skb, status,
1405                                   rx_desc->special);
1406
1407 next_desc:
1408                 rx_desc->status = 0;
1409
1410                 /* return some buffers to hardware, one at a time is too slow */
1411                 if (unlikely(cleaned_count >= E1000_RX_BUFFER_WRITE)) {
1412                         adapter->alloc_rx_buf(adapter, cleaned_count);
1413                         cleaned_count = 0;
1414                 }
1415
1416                 /* use prefetched values */
1417                 rx_desc = next_rxd;
1418                 buffer_info = next_buffer;
1419         }
1420         rx_ring->next_to_clean = i;
1421
1422         cleaned_count = e1000_desc_unused(rx_ring);
1423         if (cleaned_count)
1424                 adapter->alloc_rx_buf(adapter, cleaned_count);
1425
1426         adapter->total_rx_bytes += total_rx_bytes;
1427         adapter->total_rx_packets += total_rx_packets;
1428         return cleaned;
1429 }
1430
1431 /**
1432  * e1000_clean_rx_ring - Free Rx Buffers per Queue
1433  * @adapter: board private structure
1434  **/
1435 static void e1000_clean_rx_ring(struct e1000_adapter *adapter)
1436 {
1437         struct e1000_ring *rx_ring = adapter->rx_ring;
1438         struct e1000_buffer *buffer_info;
1439         struct e1000_ps_page *ps_page;
1440         struct pci_dev *pdev = adapter->pdev;
1441         unsigned int i, j;
1442
1443         /* Free all the Rx ring sk_buffs */
1444         for (i = 0; i < rx_ring->count; i++) {
1445                 buffer_info = &rx_ring->buffer_info[i];
1446                 if (buffer_info->dma) {
1447                         if (adapter->clean_rx == e1000_clean_rx_irq)
1448                                 dma_unmap_single(&pdev->dev, buffer_info->dma,
1449                                                  adapter->rx_buffer_len,
1450                                                  DMA_FROM_DEVICE);
1451                         else if (adapter->clean_rx == e1000_clean_jumbo_rx_irq)
1452                                 dma_unmap_page(&pdev->dev, buffer_info->dma,
1453                                                PAGE_SIZE,
1454                                                DMA_FROM_DEVICE);
1455                         else if (adapter->clean_rx == e1000_clean_rx_irq_ps)
1456                                 dma_unmap_single(&pdev->dev, buffer_info->dma,
1457                                                  adapter->rx_ps_bsize0,
1458                                                  DMA_FROM_DEVICE);
1459                         buffer_info->dma = 0;
1460                 }
1461
1462                 if (buffer_info->page) {
1463                         put_page(buffer_info->page);
1464                         buffer_info->page = NULL;
1465                 }
1466
1467                 if (buffer_info->skb) {
1468                         dev_kfree_skb(buffer_info->skb);
1469                         buffer_info->skb = NULL;
1470                 }
1471
1472                 for (j = 0; j < PS_PAGE_BUFFERS; j++) {
1473                         ps_page = &buffer_info->ps_pages[j];
1474                         if (!ps_page->page)
1475                                 break;
1476                         dma_unmap_page(&pdev->dev, ps_page->dma, PAGE_SIZE,
1477                                        DMA_FROM_DEVICE);
1478                         ps_page->dma = 0;
1479                         put_page(ps_page->page);
1480                         ps_page->page = NULL;
1481                 }
1482         }
1483
1484         /* there also may be some cached data from a chained receive */
1485         if (rx_ring->rx_skb_top) {
1486                 dev_kfree_skb(rx_ring->rx_skb_top);
1487                 rx_ring->rx_skb_top = NULL;
1488         }
1489
1490         /* Zero out the descriptor ring */
1491         memset(rx_ring->desc, 0, rx_ring->size);
1492
1493         rx_ring->next_to_clean = 0;
1494         rx_ring->next_to_use = 0;
1495         adapter->flags2 &= ~FLAG2_IS_DISCARDING;
1496
1497         writel(0, adapter->hw.hw_addr + rx_ring->head);
1498         writel(0, adapter->hw.hw_addr + rx_ring->tail);
1499 }
1500
1501 static void e1000e_downshift_workaround(struct work_struct *work)
1502 {
1503         struct e1000_adapter *adapter = container_of(work,
1504                                         struct e1000_adapter, downshift_task);
1505
1506         if (test_bit(__E1000_DOWN, &adapter->state))
1507                 return;
1508
1509         e1000e_gig_downshift_workaround_ich8lan(&adapter->hw);
1510 }
1511
1512 /**
1513  * e1000_intr_msi - Interrupt Handler
1514  * @irq: interrupt number
1515  * @data: pointer to a network interface device structure
1516  **/
1517 static irqreturn_t e1000_intr_msi(int irq, void *data)
1518 {
1519         struct net_device *netdev = data;
1520         struct e1000_adapter *adapter = netdev_priv(netdev);
1521         struct e1000_hw *hw = &adapter->hw;
1522         u32 icr = er32(ICR);
1523
1524         /*
1525          * read ICR disables interrupts using IAM
1526          */
1527
1528         if (icr & E1000_ICR_LSC) {
1529                 hw->mac.get_link_status = 1;
1530                 /*
1531                  * ICH8 workaround-- Call gig speed drop workaround on cable
1532                  * disconnect (LSC) before accessing any PHY registers
1533                  */
1534                 if ((adapter->flags & FLAG_LSC_GIG_SPEED_DROP) &&
1535                     (!(er32(STATUS) & E1000_STATUS_LU)))
1536                         schedule_work(&adapter->downshift_task);
1537
1538                 /*
1539                  * 80003ES2LAN workaround-- For packet buffer work-around on
1540                  * link down event; disable receives here in the ISR and reset
1541                  * adapter in watchdog
1542                  */
1543                 if (netif_carrier_ok(netdev) &&
1544                     adapter->flags & FLAG_RX_NEEDS_RESTART) {
1545                         /* disable receives */
1546                         u32 rctl = er32(RCTL);
1547                         ew32(RCTL, rctl & ~E1000_RCTL_EN);
1548                         adapter->flags |= FLAG_RX_RESTART_NOW;
1549                 }
1550                 /* guard against interrupt when we're going down */
1551                 if (!test_bit(__E1000_DOWN, &adapter->state))
1552                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
1553         }
1554
1555         if (napi_schedule_prep(&adapter->napi)) {
1556                 adapter->total_tx_bytes = 0;
1557                 adapter->total_tx_packets = 0;
1558                 adapter->total_rx_bytes = 0;
1559                 adapter->total_rx_packets = 0;
1560                 __napi_schedule(&adapter->napi);
1561         }
1562
1563         return IRQ_HANDLED;
1564 }
1565
1566 /**
1567  * e1000_intr - Interrupt Handler
1568  * @irq: interrupt number
1569  * @data: pointer to a network interface device structure
1570  **/
1571 static irqreturn_t e1000_intr(int irq, void *data)
1572 {
1573         struct net_device *netdev = data;
1574         struct e1000_adapter *adapter = netdev_priv(netdev);
1575         struct e1000_hw *hw = &adapter->hw;
1576         u32 rctl, icr = er32(ICR);
1577
1578         if (!icr || test_bit(__E1000_DOWN, &adapter->state))
1579                 return IRQ_NONE;  /* Not our interrupt */
1580
1581         /*
1582          * IMS will not auto-mask if INT_ASSERTED is not set, and if it is
1583          * not set, then the adapter didn't send an interrupt
1584          */
1585         if (!(icr & E1000_ICR_INT_ASSERTED))
1586                 return IRQ_NONE;
1587
1588         /*
1589          * Interrupt Auto-Mask...upon reading ICR,
1590          * interrupts are masked.  No need for the
1591          * IMC write
1592          */
1593
1594         if (icr & E1000_ICR_LSC) {
1595                 hw->mac.get_link_status = 1;
1596                 /*
1597                  * ICH8 workaround-- Call gig speed drop workaround on cable
1598                  * disconnect (LSC) before accessing any PHY registers
1599                  */
1600                 if ((adapter->flags & FLAG_LSC_GIG_SPEED_DROP) &&
1601                     (!(er32(STATUS) & E1000_STATUS_LU)))
1602                         schedule_work(&adapter->downshift_task);
1603
1604                 /*
1605                  * 80003ES2LAN workaround--
1606                  * For packet buffer work-around on link down event;
1607                  * disable receives here in the ISR and
1608                  * reset adapter in watchdog
1609                  */
1610                 if (netif_carrier_ok(netdev) &&
1611                     (adapter->flags & FLAG_RX_NEEDS_RESTART)) {
1612                         /* disable receives */
1613                         rctl = er32(RCTL);
1614                         ew32(RCTL, rctl & ~E1000_RCTL_EN);
1615                         adapter->flags |= FLAG_RX_RESTART_NOW;
1616                 }
1617                 /* guard against interrupt when we're going down */
1618                 if (!test_bit(__E1000_DOWN, &adapter->state))
1619                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
1620         }
1621
1622         if (napi_schedule_prep(&adapter->napi)) {
1623                 adapter->total_tx_bytes = 0;
1624                 adapter->total_tx_packets = 0;
1625                 adapter->total_rx_bytes = 0;
1626                 adapter->total_rx_packets = 0;
1627                 __napi_schedule(&adapter->napi);
1628         }
1629
1630         return IRQ_HANDLED;
1631 }
1632
1633 static irqreturn_t e1000_msix_other(int irq, void *data)
1634 {
1635         struct net_device *netdev = data;
1636         struct e1000_adapter *adapter = netdev_priv(netdev);
1637         struct e1000_hw *hw = &adapter->hw;
1638         u32 icr = er32(ICR);
1639
1640         if (!(icr & E1000_ICR_INT_ASSERTED)) {
1641                 if (!test_bit(__E1000_DOWN, &adapter->state))
1642                         ew32(IMS, E1000_IMS_OTHER);
1643                 return IRQ_NONE;
1644         }
1645
1646         if (icr & adapter->eiac_mask)
1647                 ew32(ICS, (icr & adapter->eiac_mask));
1648
1649         if (icr & E1000_ICR_OTHER) {
1650                 if (!(icr & E1000_ICR_LSC))
1651                         goto no_link_interrupt;
1652                 hw->mac.get_link_status = 1;
1653                 /* guard against interrupt when we're going down */
1654                 if (!test_bit(__E1000_DOWN, &adapter->state))
1655                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
1656         }
1657
1658 no_link_interrupt:
1659         if (!test_bit(__E1000_DOWN, &adapter->state))
1660                 ew32(IMS, E1000_IMS_LSC | E1000_IMS_OTHER);
1661
1662         return IRQ_HANDLED;
1663 }
1664
1665
1666 static irqreturn_t e1000_intr_msix_tx(int irq, void *data)
1667 {
1668         struct net_device *netdev = data;
1669         struct e1000_adapter *adapter = netdev_priv(netdev);
1670         struct e1000_hw *hw = &adapter->hw;
1671         struct e1000_ring *tx_ring = adapter->tx_ring;
1672
1673
1674         adapter->total_tx_bytes = 0;
1675         adapter->total_tx_packets = 0;
1676
1677         if (!e1000_clean_tx_irq(adapter))
1678                 /* Ring was not completely cleaned, so fire another interrupt */
1679                 ew32(ICS, tx_ring->ims_val);
1680
1681         return IRQ_HANDLED;
1682 }
1683
1684 static irqreturn_t e1000_intr_msix_rx(int irq, void *data)
1685 {
1686         struct net_device *netdev = data;
1687         struct e1000_adapter *adapter = netdev_priv(netdev);
1688
1689         /* Write the ITR value calculated at the end of the
1690          * previous interrupt.
1691          */
1692         if (adapter->rx_ring->set_itr) {
1693                 writel(1000000000 / (adapter->rx_ring->itr_val * 256),
1694                        adapter->hw.hw_addr + adapter->rx_ring->itr_register);
1695                 adapter->rx_ring->set_itr = 0;
1696         }
1697
1698         if (napi_schedule_prep(&adapter->napi)) {
1699                 adapter->total_rx_bytes = 0;
1700                 adapter->total_rx_packets = 0;
1701                 __napi_schedule(&adapter->napi);
1702         }
1703         return IRQ_HANDLED;
1704 }
1705
1706 /**
1707  * e1000_configure_msix - Configure MSI-X hardware
1708  *
1709  * e1000_configure_msix sets up the hardware to properly
1710  * generate MSI-X interrupts.
1711  **/
1712 static void e1000_configure_msix(struct e1000_adapter *adapter)
1713 {
1714         struct e1000_hw *hw = &adapter->hw;
1715         struct e1000_ring *rx_ring = adapter->rx_ring;
1716         struct e1000_ring *tx_ring = adapter->tx_ring;
1717         int vector = 0;
1718         u32 ctrl_ext, ivar = 0;
1719
1720         adapter->eiac_mask = 0;
1721
1722         /* Workaround issue with spurious interrupts on 82574 in MSI-X mode */
1723         if (hw->mac.type == e1000_82574) {
1724                 u32 rfctl = er32(RFCTL);
1725                 rfctl |= E1000_RFCTL_ACK_DIS;
1726                 ew32(RFCTL, rfctl);
1727         }
1728
1729 #define E1000_IVAR_INT_ALLOC_VALID      0x8
1730         /* Configure Rx vector */
1731         rx_ring->ims_val = E1000_IMS_RXQ0;
1732         adapter->eiac_mask |= rx_ring->ims_val;
1733         if (rx_ring->itr_val)
1734                 writel(1000000000 / (rx_ring->itr_val * 256),
1735                        hw->hw_addr + rx_ring->itr_register);
1736         else
1737                 writel(1, hw->hw_addr + rx_ring->itr_register);
1738         ivar = E1000_IVAR_INT_ALLOC_VALID | vector;
1739
1740         /* Configure Tx vector */
1741         tx_ring->ims_val = E1000_IMS_TXQ0;
1742         vector++;
1743         if (tx_ring->itr_val)
1744                 writel(1000000000 / (tx_ring->itr_val * 256),
1745                        hw->hw_addr + tx_ring->itr_register);
1746         else
1747                 writel(1, hw->hw_addr + tx_ring->itr_register);
1748         adapter->eiac_mask |= tx_ring->ims_val;
1749         ivar |= ((E1000_IVAR_INT_ALLOC_VALID | vector) << 8);
1750
1751         /* set vector for Other Causes, e.g. link changes */
1752         vector++;
1753         ivar |= ((E1000_IVAR_INT_ALLOC_VALID | vector) << 16);
1754         if (rx_ring->itr_val)
1755                 writel(1000000000 / (rx_ring->itr_val * 256),
1756                        hw->hw_addr + E1000_EITR_82574(vector));
1757         else
1758                 writel(1, hw->hw_addr + E1000_EITR_82574(vector));
1759
1760         /* Cause Tx interrupts on every write back */
1761         ivar |= (1 << 31);
1762
1763         ew32(IVAR, ivar);
1764
1765         /* enable MSI-X PBA support */
1766         ctrl_ext = er32(CTRL_EXT);
1767         ctrl_ext |= E1000_CTRL_EXT_PBA_CLR;
1768
1769         /* Auto-Mask Other interrupts upon ICR read */
1770 #define E1000_EIAC_MASK_82574   0x01F00000
1771         ew32(IAM, ~E1000_EIAC_MASK_82574 | E1000_IMS_OTHER);
1772         ctrl_ext |= E1000_CTRL_EXT_EIAME;
1773         ew32(CTRL_EXT, ctrl_ext);
1774         e1e_flush();
1775 }
1776
1777 void e1000e_reset_interrupt_capability(struct e1000_adapter *adapter)
1778 {
1779         if (adapter->msix_entries) {
1780                 pci_disable_msix(adapter->pdev);
1781                 kfree(adapter->msix_entries);
1782                 adapter->msix_entries = NULL;
1783         } else if (adapter->flags & FLAG_MSI_ENABLED) {
1784                 pci_disable_msi(adapter->pdev);
1785                 adapter->flags &= ~FLAG_MSI_ENABLED;
1786         }
1787 }
1788
1789 /**
1790  * e1000e_set_interrupt_capability - set MSI or MSI-X if supported
1791  *
1792  * Attempt to configure interrupts using the best available
1793  * capabilities of the hardware and kernel.
1794  **/
1795 void e1000e_set_interrupt_capability(struct e1000_adapter *adapter)
1796 {
1797         int err;
1798         int i;
1799
1800         switch (adapter->int_mode) {
1801         case E1000E_INT_MODE_MSIX:
1802                 if (adapter->flags & FLAG_HAS_MSIX) {
1803                         adapter->num_vectors = 3; /* RxQ0, TxQ0 and other */
1804                         adapter->msix_entries = kcalloc(adapter->num_vectors,
1805                                                       sizeof(struct msix_entry),
1806                                                       GFP_KERNEL);
1807                         if (adapter->msix_entries) {
1808                                 for (i = 0; i < adapter->num_vectors; i++)
1809                                         adapter->msix_entries[i].entry = i;
1810
1811                                 err = pci_enable_msix(adapter->pdev,
1812                                                       adapter->msix_entries,
1813                                                       adapter->num_vectors);
1814                                 if (err == 0)
1815                                         return;
1816                         }
1817                         /* MSI-X failed, so fall through and try MSI */
1818                         e_err("Failed to initialize MSI-X interrupts.  "
1819                               "Falling back to MSI interrupts.\n");
1820                         e1000e_reset_interrupt_capability(adapter);
1821                 }
1822                 adapter->int_mode = E1000E_INT_MODE_MSI;
1823                 /* Fall through */
1824         case E1000E_INT_MODE_MSI:
1825                 if (!pci_enable_msi(adapter->pdev)) {
1826                         adapter->flags |= FLAG_MSI_ENABLED;
1827                 } else {
1828                         adapter->int_mode = E1000E_INT_MODE_LEGACY;
1829                         e_err("Failed to initialize MSI interrupts.  Falling "
1830                               "back to legacy interrupts.\n");
1831                 }
1832                 /* Fall through */
1833         case E1000E_INT_MODE_LEGACY:
1834                 /* Don't do anything; this is the system default */
1835                 break;
1836         }
1837
1838         /* store the number of vectors being used */
1839         adapter->num_vectors = 1;
1840 }
1841
1842 /**
1843  * e1000_request_msix - Initialize MSI-X interrupts
1844  *
1845  * e1000_request_msix allocates MSI-X vectors and requests interrupts from the
1846  * kernel.
1847  **/
1848 static int e1000_request_msix(struct e1000_adapter *adapter)
1849 {
1850         struct net_device *netdev = adapter->netdev;
1851         int err = 0, vector = 0;
1852
1853         if (strlen(netdev->name) < (IFNAMSIZ - 5))
1854                 snprintf(adapter->rx_ring->name,
1855                          sizeof(adapter->rx_ring->name) - 1,
1856                          "%s-rx-0", netdev->name);
1857         else
1858                 memcpy(adapter->rx_ring->name, netdev->name, IFNAMSIZ);
1859         err = request_irq(adapter->msix_entries[vector].vector,
1860                           e1000_intr_msix_rx, 0, adapter->rx_ring->name,
1861                           netdev);
1862         if (err)
1863                 goto out;
1864         adapter->rx_ring->itr_register = E1000_EITR_82574(vector);
1865         adapter->rx_ring->itr_val = adapter->itr;
1866         vector++;
1867
1868         if (strlen(netdev->name) < (IFNAMSIZ - 5))
1869                 snprintf(adapter->tx_ring->name,
1870                          sizeof(adapter->tx_ring->name) - 1,
1871                          "%s-tx-0", netdev->name);
1872         else
1873                 memcpy(adapter->tx_ring->name, netdev->name, IFNAMSIZ);
1874         err = request_irq(adapter->msix_entries[vector].vector,
1875                           e1000_intr_msix_tx, 0, adapter->tx_ring->name,
1876                           netdev);
1877         if (err)
1878                 goto out;
1879         adapter->tx_ring->itr_register = E1000_EITR_82574(vector);
1880         adapter->tx_ring->itr_val = adapter->itr;
1881         vector++;
1882
1883         err = request_irq(adapter->msix_entries[vector].vector,
1884                           e1000_msix_other, 0, netdev->name, netdev);
1885         if (err)
1886                 goto out;
1887
1888         e1000_configure_msix(adapter);
1889         return 0;
1890 out:
1891         return err;
1892 }
1893
1894 /**
1895  * e1000_request_irq - initialize interrupts
1896  *
1897  * Attempts to configure interrupts using the best available
1898  * capabilities of the hardware and kernel.
1899  **/
1900 static int e1000_request_irq(struct e1000_adapter *adapter)
1901 {
1902         struct net_device *netdev = adapter->netdev;
1903         int err;
1904
1905         if (adapter->msix_entries) {
1906                 err = e1000_request_msix(adapter);
1907                 if (!err)
1908                         return err;
1909                 /* fall back to MSI */
1910                 e1000e_reset_interrupt_capability(adapter);
1911                 adapter->int_mode = E1000E_INT_MODE_MSI;
1912                 e1000e_set_interrupt_capability(adapter);
1913         }
1914         if (adapter->flags & FLAG_MSI_ENABLED) {
1915                 err = request_irq(adapter->pdev->irq, e1000_intr_msi, 0,
1916                                   netdev->name, netdev);
1917                 if (!err)
1918                         return err;
1919
1920                 /* fall back to legacy interrupt */
1921                 e1000e_reset_interrupt_capability(adapter);
1922                 adapter->int_mode = E1000E_INT_MODE_LEGACY;
1923         }
1924
1925         err = request_irq(adapter->pdev->irq, e1000_intr, IRQF_SHARED,
1926                           netdev->name, netdev);
1927         if (err)
1928                 e_err("Unable to allocate interrupt, Error: %d\n", err);
1929
1930         return err;
1931 }
1932
1933 static void e1000_free_irq(struct e1000_adapter *adapter)
1934 {
1935         struct net_device *netdev = adapter->netdev;
1936
1937         if (adapter->msix_entries) {
1938                 int vector = 0;
1939
1940                 free_irq(adapter->msix_entries[vector].vector, netdev);
1941                 vector++;
1942
1943                 free_irq(adapter->msix_entries[vector].vector, netdev);
1944                 vector++;
1945
1946                 /* Other Causes interrupt vector */
1947                 free_irq(adapter->msix_entries[vector].vector, netdev);
1948                 return;
1949         }
1950
1951         free_irq(adapter->pdev->irq, netdev);
1952 }
1953
1954 /**
1955  * e1000_irq_disable - Mask off interrupt generation on the NIC
1956  **/
1957 static void e1000_irq_disable(struct e1000_adapter *adapter)
1958 {
1959         struct e1000_hw *hw = &adapter->hw;
1960
1961         ew32(IMC, ~0);
1962         if (adapter->msix_entries)
1963                 ew32(EIAC_82574, 0);
1964         e1e_flush();
1965
1966         if (adapter->msix_entries) {
1967                 int i;
1968                 for (i = 0; i < adapter->num_vectors; i++)
1969                         synchronize_irq(adapter->msix_entries[i].vector);
1970         } else {
1971                 synchronize_irq(adapter->pdev->irq);
1972         }
1973 }
1974
1975 /**
1976  * e1000_irq_enable - Enable default interrupt generation settings
1977  **/
1978 static void e1000_irq_enable(struct e1000_adapter *adapter)
1979 {
1980         struct e1000_hw *hw = &adapter->hw;
1981
1982         if (adapter->msix_entries) {
1983                 ew32(EIAC_82574, adapter->eiac_mask & E1000_EIAC_MASK_82574);
1984                 ew32(IMS, adapter->eiac_mask | E1000_IMS_OTHER | E1000_IMS_LSC);
1985         } else {
1986                 ew32(IMS, IMS_ENABLE_MASK);
1987         }
1988         e1e_flush();
1989 }
1990
1991 /**
1992  * e1000e_get_hw_control - get control of the h/w from f/w
1993  * @adapter: address of board private structure
1994  *
1995  * e1000e_get_hw_control sets {CTRL_EXT|SWSM}:DRV_LOAD bit.
1996  * For ASF and Pass Through versions of f/w this means that
1997  * the driver is loaded. For AMT version (only with 82573)
1998  * of the f/w this means that the network i/f is open.
1999  **/
2000 void e1000e_get_hw_control(struct e1000_adapter *adapter)
2001 {
2002         struct e1000_hw *hw = &adapter->hw;
2003         u32 ctrl_ext;
2004         u32 swsm;
2005
2006         /* Let firmware know the driver has taken over */
2007         if (adapter->flags & FLAG_HAS_SWSM_ON_LOAD) {
2008                 swsm = er32(SWSM);
2009                 ew32(SWSM, swsm | E1000_SWSM_DRV_LOAD);
2010         } else if (adapter->flags & FLAG_HAS_CTRLEXT_ON_LOAD) {
2011                 ctrl_ext = er32(CTRL_EXT);
2012                 ew32(CTRL_EXT, ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
2013         }
2014 }
2015
2016 /**
2017  * e1000e_release_hw_control - release control of the h/w to f/w
2018  * @adapter: address of board private structure
2019  *
2020  * e1000e_release_hw_control resets {CTRL_EXT|SWSM}:DRV_LOAD bit.
2021  * For ASF and Pass Through versions of f/w this means that the
2022  * driver is no longer loaded. For AMT version (only with 82573) i
2023  * of the f/w this means that the network i/f is closed.
2024  *
2025  **/
2026 void e1000e_release_hw_control(struct e1000_adapter *adapter)
2027 {
2028         struct e1000_hw *hw = &adapter->hw;
2029         u32 ctrl_ext;
2030         u32 swsm;
2031
2032         /* Let firmware taken over control of h/w */
2033         if (adapter->flags & FLAG_HAS_SWSM_ON_LOAD) {
2034                 swsm = er32(SWSM);
2035                 ew32(SWSM, swsm & ~E1000_SWSM_DRV_LOAD);
2036         } else if (adapter->flags & FLAG_HAS_CTRLEXT_ON_LOAD) {
2037                 ctrl_ext = er32(CTRL_EXT);
2038                 ew32(CTRL_EXT, ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
2039         }
2040 }
2041
2042 /**
2043  * @e1000_alloc_ring - allocate memory for a ring structure
2044  **/
2045 static int e1000_alloc_ring_dma(struct e1000_adapter *adapter,
2046                                 struct e1000_ring *ring)
2047 {
2048         struct pci_dev *pdev = adapter->pdev;
2049
2050         ring->desc = dma_alloc_coherent(&pdev->dev, ring->size, &ring->dma,
2051                                         GFP_KERNEL);
2052         if (!ring->desc)
2053                 return -ENOMEM;
2054
2055         return 0;
2056 }
2057
2058 /**
2059  * e1000e_setup_tx_resources - allocate Tx resources (Descriptors)
2060  * @adapter: board private structure
2061  *
2062  * Return 0 on success, negative on failure
2063  **/
2064 int e1000e_setup_tx_resources(struct e1000_adapter *adapter)
2065 {
2066         struct e1000_ring *tx_ring = adapter->tx_ring;
2067         int err = -ENOMEM, size;
2068
2069         size = sizeof(struct e1000_buffer) * tx_ring->count;
2070         tx_ring->buffer_info = vzalloc(size);
2071         if (!tx_ring->buffer_info)
2072                 goto err;
2073
2074         /* round up to nearest 4K */
2075         tx_ring->size = tx_ring->count * sizeof(struct e1000_tx_desc);
2076         tx_ring->size = ALIGN(tx_ring->size, 4096);
2077
2078         err = e1000_alloc_ring_dma(adapter, tx_ring);
2079         if (err)
2080                 goto err;
2081
2082         tx_ring->next_to_use = 0;
2083         tx_ring->next_to_clean = 0;
2084
2085         return 0;
2086 err:
2087         vfree(tx_ring->buffer_info);
2088         e_err("Unable to allocate memory for the transmit descriptor ring\n");
2089         return err;
2090 }
2091
2092 /**
2093  * e1000e_setup_rx_resources - allocate Rx resources (Descriptors)
2094  * @adapter: board private structure
2095  *
2096  * Returns 0 on success, negative on failure
2097  **/
2098 int e1000e_setup_rx_resources(struct e1000_adapter *adapter)
2099 {
2100         struct e1000_ring *rx_ring = adapter->rx_ring;
2101         struct e1000_buffer *buffer_info;
2102         int i, size, desc_len, err = -ENOMEM;
2103
2104         size = sizeof(struct e1000_buffer) * rx_ring->count;
2105         rx_ring->buffer_info = vzalloc(size);
2106         if (!rx_ring->buffer_info)
2107                 goto err;
2108
2109         for (i = 0; i < rx_ring->count; i++) {
2110                 buffer_info = &rx_ring->buffer_info[i];
2111                 buffer_info->ps_pages = kcalloc(PS_PAGE_BUFFERS,
2112                                                 sizeof(struct e1000_ps_page),
2113                                                 GFP_KERNEL);
2114                 if (!buffer_info->ps_pages)
2115                         goto err_pages;
2116         }
2117
2118         desc_len = sizeof(union e1000_rx_desc_packet_split);
2119
2120         /* Round up to nearest 4K */
2121         rx_ring->size = rx_ring->count * desc_len;
2122         rx_ring->size = ALIGN(rx_ring->size, 4096);
2123
2124         err = e1000_alloc_ring_dma(adapter, rx_ring);
2125         if (err)
2126                 goto err_pages;
2127
2128         rx_ring->next_to_clean = 0;
2129         rx_ring->next_to_use = 0;
2130         rx_ring->rx_skb_top = NULL;
2131
2132         return 0;
2133
2134 err_pages:
2135         for (i = 0; i < rx_ring->count; i++) {
2136                 buffer_info = &rx_ring->buffer_info[i];
2137                 kfree(buffer_info->ps_pages);
2138         }
2139 err:
2140         vfree(rx_ring->buffer_info);
2141         e_err("Unable to allocate memory for the receive descriptor ring\n");
2142         return err;
2143 }
2144
2145 /**
2146  * e1000_clean_tx_ring - Free Tx Buffers
2147  * @adapter: board private structure
2148  **/
2149 static void e1000_clean_tx_ring(struct e1000_adapter *adapter)
2150 {
2151         struct e1000_ring *tx_ring = adapter->tx_ring;
2152         struct e1000_buffer *buffer_info;
2153         unsigned long size;
2154         unsigned int i;
2155
2156         for (i = 0; i < tx_ring->count; i++) {
2157                 buffer_info = &tx_ring->buffer_info[i];
2158                 e1000_put_txbuf(adapter, buffer_info);
2159         }
2160
2161         size = sizeof(struct e1000_buffer) * tx_ring->count;
2162         memset(tx_ring->buffer_info, 0, size);
2163
2164         memset(tx_ring->desc, 0, tx_ring->size);
2165
2166         tx_ring->next_to_use = 0;
2167         tx_ring->next_to_clean = 0;
2168
2169         writel(0, adapter->hw.hw_addr + tx_ring->head);
2170         writel(0, adapter->hw.hw_addr + tx_ring->tail);
2171 }
2172
2173 /**
2174  * e1000e_free_tx_resources - Free Tx Resources per Queue
2175  * @adapter: board private structure
2176  *
2177  * Free all transmit software resources
2178  **/
2179 void e1000e_free_tx_resources(struct e1000_adapter *adapter)
2180 {
2181         struct pci_dev *pdev = adapter->pdev;
2182         struct e1000_ring *tx_ring = adapter->tx_ring;
2183
2184         e1000_clean_tx_ring(adapter);
2185
2186         vfree(tx_ring->buffer_info);
2187         tx_ring->buffer_info = NULL;
2188
2189         dma_free_coherent(&pdev->dev, tx_ring->size, tx_ring->desc,
2190                           tx_ring->dma);
2191         tx_ring->desc = NULL;
2192 }
2193
2194 /**
2195  * e1000e_free_rx_resources - Free Rx Resources
2196  * @adapter: board private structure
2197  *
2198  * Free all receive software resources
2199  **/
2200
2201 void e1000e_free_rx_resources(struct e1000_adapter *adapter)
2202 {
2203         struct pci_dev *pdev = adapter->pdev;
2204         struct e1000_ring *rx_ring = adapter->rx_ring;
2205         int i;
2206
2207         e1000_clean_rx_ring(adapter);
2208
2209         for (i = 0; i < rx_ring->count; i++)
2210                 kfree(rx_ring->buffer_info[i].ps_pages);
2211
2212         vfree(rx_ring->buffer_info);
2213         rx_ring->buffer_info = NULL;
2214
2215         dma_free_coherent(&pdev->dev, rx_ring->size, rx_ring->desc,
2216                           rx_ring->dma);
2217         rx_ring->desc = NULL;
2218 }
2219
2220 /**
2221  * e1000_update_itr - update the dynamic ITR value based on statistics
2222  * @adapter: pointer to adapter
2223  * @itr_setting: current adapter->itr
2224  * @packets: the number of packets during this measurement interval
2225  * @bytes: the number of bytes during this measurement interval
2226  *
2227  *      Stores a new ITR value based on packets and byte
2228  *      counts during the last interrupt.  The advantage of per interrupt
2229  *      computation is faster updates and more accurate ITR for the current
2230  *      traffic pattern.  Constants in this function were computed
2231  *      based on theoretical maximum wire speed and thresholds were set based
2232  *      on testing data as well as attempting to minimize response time
2233  *      while increasing bulk throughput.  This functionality is controlled
2234  *      by the InterruptThrottleRate module parameter.
2235  **/
2236 static unsigned int e1000_update_itr(struct e1000_adapter *adapter,
2237                                      u16 itr_setting, int packets,
2238                                      int bytes)
2239 {
2240         unsigned int retval = itr_setting;
2241
2242         if (packets == 0)
2243                 goto update_itr_done;
2244
2245         switch (itr_setting) {
2246         case lowest_latency:
2247                 /* handle TSO and jumbo frames */
2248                 if (bytes/packets > 8000)
2249                         retval = bulk_latency;
2250                 else if ((packets < 5) && (bytes > 512))
2251                         retval = low_latency;
2252                 break;
2253         case low_latency:  /* 50 usec aka 20000 ints/s */
2254                 if (bytes > 10000) {
2255                         /* this if handles the TSO accounting */
2256                         if (bytes/packets > 8000)
2257                                 retval = bulk_latency;
2258                         else if ((packets < 10) || ((bytes/packets) > 1200))
2259                                 retval = bulk_latency;
2260                         else if ((packets > 35))
2261                                 retval = lowest_latency;
2262                 } else if (bytes/packets > 2000) {
2263                         retval = bulk_latency;
2264                 } else if (packets <= 2 && bytes < 512) {
2265                         retval = lowest_latency;
2266                 }
2267                 break;
2268         case bulk_latency: /* 250 usec aka 4000 ints/s */
2269                 if (bytes > 25000) {
2270                         if (packets > 35)
2271                                 retval = low_latency;
2272                 } else if (bytes < 6000) {
2273                         retval = low_latency;
2274                 }
2275                 break;
2276         }
2277
2278 update_itr_done:
2279         return retval;
2280 }
2281
2282 static void e1000_set_itr(struct e1000_adapter *adapter)
2283 {
2284         struct e1000_hw *hw = &adapter->hw;
2285         u16 current_itr;
2286         u32 new_itr = adapter->itr;
2287
2288         /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
2289         if (adapter->link_speed != SPEED_1000) {
2290                 current_itr = 0;
2291                 new_itr = 4000;
2292                 goto set_itr_now;
2293         }
2294
2295         if (adapter->flags2 & FLAG2_DISABLE_AIM) {
2296                 new_itr = 0;
2297                 goto set_itr_now;
2298         }
2299
2300         adapter->tx_itr = e1000_update_itr(adapter,
2301                                     adapter->tx_itr,
2302                                     adapter->total_tx_packets,
2303                                     adapter->total_tx_bytes);
2304         /* conservative mode (itr 3) eliminates the lowest_latency setting */
2305         if (adapter->itr_setting == 3 && adapter->tx_itr == lowest_latency)
2306                 adapter->tx_itr = low_latency;
2307
2308         adapter->rx_itr = e1000_update_itr(adapter,
2309                                     adapter->rx_itr,
2310                                     adapter->total_rx_packets,
2311                                     adapter->total_rx_bytes);
2312         /* conservative mode (itr 3) eliminates the lowest_latency setting */
2313         if (adapter->itr_setting == 3 && adapter->rx_itr == lowest_latency)
2314                 adapter->rx_itr = low_latency;
2315
2316         current_itr = max(adapter->rx_itr, adapter->tx_itr);
2317
2318         switch (current_itr) {
2319         /* counts and packets in update_itr are dependent on these numbers */
2320         case lowest_latency:
2321                 new_itr = 70000;
2322                 break;
2323         case low_latency:
2324                 new_itr = 20000; /* aka hwitr = ~200 */
2325                 break;
2326         case bulk_latency:
2327                 new_itr = 4000;
2328                 break;
2329         default:
2330                 break;
2331         }
2332
2333 set_itr_now:
2334         if (new_itr != adapter->itr) {
2335                 /*
2336                  * this attempts to bias the interrupt rate towards Bulk
2337                  * by adding intermediate steps when interrupt rate is
2338                  * increasing
2339                  */
2340                 new_itr = new_itr > adapter->itr ?
2341                              min(adapter->itr + (new_itr >> 2), new_itr) :
2342                              new_itr;
2343                 adapter->itr = new_itr;
2344                 adapter->rx_ring->itr_val = new_itr;
2345                 if (adapter->msix_entries)
2346                         adapter->rx_ring->set_itr = 1;
2347                 else
2348                         if (new_itr)
2349                                 ew32(ITR, 1000000000 / (new_itr * 256));
2350                         else
2351                                 ew32(ITR, 0);
2352         }
2353 }
2354
2355 /**
2356  * e1000_alloc_queues - Allocate memory for all rings
2357  * @adapter: board private structure to initialize
2358  **/
2359 static int __devinit e1000_alloc_queues(struct e1000_adapter *adapter)
2360 {
2361         adapter->tx_ring = kzalloc(sizeof(struct e1000_ring), GFP_KERNEL);
2362         if (!adapter->tx_ring)
2363                 goto err;
2364
2365         adapter->rx_ring = kzalloc(sizeof(struct e1000_ring), GFP_KERNEL);
2366         if (!adapter->rx_ring)
2367                 goto err;
2368
2369         return 0;
2370 err:
2371         e_err("Unable to allocate memory for queues\n");
2372         kfree(adapter->rx_ring);
2373         kfree(adapter->tx_ring);
2374         return -ENOMEM;
2375 }
2376
2377 /**
2378  * e1000_clean - NAPI Rx polling callback
2379  * @napi: struct associated with this polling callback
2380  * @budget: amount of packets driver is allowed to process this poll
2381  **/
2382 static int e1000_clean(struct napi_struct *napi, int budget)
2383 {
2384         struct e1000_adapter *adapter = container_of(napi, struct e1000_adapter, napi);
2385         struct e1000_hw *hw = &adapter->hw;
2386         struct net_device *poll_dev = adapter->netdev;
2387         int tx_cleaned = 1, work_done = 0;
2388
2389         adapter = netdev_priv(poll_dev);
2390
2391         if (adapter->msix_entries &&
2392             !(adapter->rx_ring->ims_val & adapter->tx_ring->ims_val))
2393                 goto clean_rx;
2394
2395         tx_cleaned = e1000_clean_tx_irq(adapter);
2396
2397 clean_rx:
2398         adapter->clean_rx(adapter, &work_done, budget);
2399
2400         if (!tx_cleaned)
2401                 work_done = budget;
2402
2403         /* If budget not fully consumed, exit the polling mode */
2404         if (work_done < budget) {
2405                 if (adapter->itr_setting & 3)
2406                         e1000_set_itr(adapter);
2407                 napi_complete(napi);
2408                 if (!test_bit(__E1000_DOWN, &adapter->state)) {
2409                         if (adapter->msix_entries)
2410                                 ew32(IMS, adapter->rx_ring->ims_val);
2411                         else
2412                                 e1000_irq_enable(adapter);
2413                 }
2414         }
2415
2416         return work_done;
2417 }
2418
2419 static void e1000_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
2420 {
2421         struct e1000_adapter *adapter = netdev_priv(netdev);
2422         struct e1000_hw *hw = &adapter->hw;
2423         u32 vfta, index;
2424
2425         /* don't update vlan cookie if already programmed */
2426         if ((adapter->hw.mng_cookie.status &
2427              E1000_MNG_DHCP_COOKIE_STATUS_VLAN) &&
2428             (vid == adapter->mng_vlan_id))
2429                 return;
2430
2431         /* add VID to filter table */
2432         if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) {
2433                 index = (vid >> 5) & 0x7F;
2434                 vfta = E1000_READ_REG_ARRAY(hw, E1000_VFTA, index);
2435                 vfta |= (1 << (vid & 0x1F));
2436                 hw->mac.ops.write_vfta(hw, index, vfta);
2437         }
2438
2439         set_bit(vid, adapter->active_vlans);
2440 }
2441
2442 static void e1000_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
2443 {
2444         struct e1000_adapter *adapter = netdev_priv(netdev);
2445         struct e1000_hw *hw = &adapter->hw;
2446         u32 vfta, index;
2447
2448         if ((adapter->hw.mng_cookie.status &
2449              E1000_MNG_DHCP_COOKIE_STATUS_VLAN) &&
2450             (vid == adapter->mng_vlan_id)) {
2451                 /* release control to f/w */
2452                 e1000e_release_hw_control(adapter);
2453                 return;
2454         }
2455
2456         /* remove VID from filter table */
2457         if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) {
2458                 index = (vid >> 5) & 0x7F;
2459                 vfta = E1000_READ_REG_ARRAY(hw, E1000_VFTA, index);
2460                 vfta &= ~(1 << (vid & 0x1F));
2461                 hw->mac.ops.write_vfta(hw, index, vfta);
2462         }
2463
2464         clear_bit(vid, adapter->active_vlans);
2465 }
2466
2467 /**
2468  * e1000e_vlan_filter_disable - helper to disable hw VLAN filtering
2469  * @adapter: board private structure to initialize
2470  **/
2471 static void e1000e_vlan_filter_disable(struct e1000_adapter *adapter)
2472 {
2473         struct net_device *netdev = adapter->netdev;
2474         struct e1000_hw *hw = &adapter->hw;
2475         u32 rctl;
2476
2477         if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) {
2478                 /* disable VLAN receive filtering */
2479                 rctl = er32(RCTL);
2480                 rctl &= ~(E1000_RCTL_VFE | E1000_RCTL_CFIEN);
2481                 ew32(RCTL, rctl);
2482
2483                 if (adapter->mng_vlan_id != (u16)E1000_MNG_VLAN_NONE) {
2484                         e1000_vlan_rx_kill_vid(netdev, adapter->mng_vlan_id);
2485                         adapter->mng_vlan_id = E1000_MNG_VLAN_NONE;
2486                 }
2487         }
2488 }
2489
2490 /**
2491  * e1000e_vlan_filter_enable - helper to enable HW VLAN filtering
2492  * @adapter: board private structure to initialize
2493  **/
2494 static void e1000e_vlan_filter_enable(struct e1000_adapter *adapter)
2495 {
2496         struct e1000_hw *hw = &adapter->hw;
2497         u32 rctl;
2498
2499         if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) {
2500                 /* enable VLAN receive filtering */
2501                 rctl = er32(RCTL);
2502                 rctl |= E1000_RCTL_VFE;
2503                 rctl &= ~E1000_RCTL_CFIEN;
2504                 ew32(RCTL, rctl);
2505         }
2506 }
2507
2508 /**
2509  * e1000e_vlan_strip_enable - helper to disable HW VLAN stripping
2510  * @adapter: board private structure to initialize
2511  **/
2512 static void e1000e_vlan_strip_disable(struct e1000_adapter *adapter)
2513 {
2514         struct e1000_hw *hw = &adapter->hw;
2515         u32 ctrl;
2516
2517         /* disable VLAN tag insert/strip */
2518         ctrl = er32(CTRL);
2519         ctrl &= ~E1000_CTRL_VME;
2520         ew32(CTRL, ctrl);
2521 }
2522
2523 /**
2524  * e1000e_vlan_strip_enable - helper to enable HW VLAN stripping
2525  * @adapter: board private structure to initialize
2526  **/
2527 static void e1000e_vlan_strip_enable(struct e1000_adapter *adapter)
2528 {
2529         struct e1000_hw *hw = &adapter->hw;
2530         u32 ctrl;
2531
2532         /* enable VLAN tag insert/strip */
2533         ctrl = er32(CTRL);
2534         ctrl |= E1000_CTRL_VME;
2535         ew32(CTRL, ctrl);
2536 }
2537
2538 static void e1000_update_mng_vlan(struct e1000_adapter *adapter)
2539 {
2540         struct net_device *netdev = adapter->netdev;
2541         u16 vid = adapter->hw.mng_cookie.vlan_id;
2542         u16 old_vid = adapter->mng_vlan_id;
2543
2544         if (adapter->hw.mng_cookie.status &
2545             E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
2546                 e1000_vlan_rx_add_vid(netdev, vid);
2547                 adapter->mng_vlan_id = vid;
2548         }
2549
2550         if ((old_vid != (u16)E1000_MNG_VLAN_NONE) && (vid != old_vid))
2551                 e1000_vlan_rx_kill_vid(netdev, old_vid);
2552 }
2553
2554 static void e1000_restore_vlan(struct e1000_adapter *adapter)
2555 {
2556         u16 vid;
2557
2558         e1000_vlan_rx_add_vid(adapter->netdev, 0);
2559
2560         for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
2561                 e1000_vlan_rx_add_vid(adapter->netdev, vid);
2562 }
2563
2564 static void e1000_init_manageability_pt(struct e1000_adapter *adapter)
2565 {
2566         struct e1000_hw *hw = &adapter->hw;
2567         u32 manc, manc2h, mdef, i, j;
2568
2569         if (!(adapter->flags & FLAG_MNG_PT_ENABLED))
2570                 return;
2571
2572         manc = er32(MANC);
2573
2574         /*
2575          * enable receiving management packets to the host. this will probably
2576          * generate destination unreachable messages from the host OS, but
2577          * the packets will be handled on SMBUS
2578          */
2579         manc |= E1000_MANC_EN_MNG2HOST;
2580         manc2h = er32(MANC2H);
2581
2582         switch (hw->mac.type) {
2583         default:
2584                 manc2h |= (E1000_MANC2H_PORT_623 | E1000_MANC2H_PORT_664);
2585                 break;
2586         case e1000_82574:
2587         case e1000_82583:
2588                 /*
2589                  * Check if IPMI pass-through decision filter already exists;
2590                  * if so, enable it.
2591                  */
2592                 for (i = 0, j = 0; i < 8; i++) {
2593                         mdef = er32(MDEF(i));
2594
2595                         /* Ignore filters with anything other than IPMI ports */
2596                         if (mdef & ~(E1000_MDEF_PORT_623 | E1000_MDEF_PORT_664))
2597                                 continue;
2598
2599                         /* Enable this decision filter in MANC2H */
2600                         if (mdef)
2601                                 manc2h |= (1 << i);
2602
2603                         j |= mdef;
2604                 }
2605
2606                 if (j == (E1000_MDEF_PORT_623 | E1000_MDEF_PORT_664))
2607                         break;
2608
2609                 /* Create new decision filter in an empty filter */
2610                 for (i = 0, j = 0; i < 8; i++)
2611                         if (er32(MDEF(i)) == 0) {
2612                                 ew32(MDEF(i), (E1000_MDEF_PORT_623 |
2613                                                E1000_MDEF_PORT_664));
2614                                 manc2h |= (1 << 1);
2615                                 j++;
2616                                 break;
2617                         }
2618
2619                 if (!j)
2620                         e_warn("Unable to create IPMI pass-through filter\n");
2621                 break;
2622         }
2623
2624         ew32(MANC2H, manc2h);
2625         ew32(MANC, manc);
2626 }
2627
2628 /**
2629  * e1000_configure_tx - Configure Transmit Unit after Reset
2630  * @adapter: board private structure
2631  *
2632  * Configure the Tx unit of the MAC after a reset.
2633  **/
2634 static void e1000_configure_tx(struct e1000_adapter *adapter)
2635 {
2636         struct e1000_hw *hw = &adapter->hw;
2637         struct e1000_ring *tx_ring = adapter->tx_ring;
2638         u64 tdba;
2639         u32 tdlen, tctl, tipg, tarc;
2640         u32 ipgr1, ipgr2;
2641
2642         /* Setup the HW Tx Head and Tail descriptor pointers */
2643         tdba = tx_ring->dma;
2644         tdlen = tx_ring->count * sizeof(struct e1000_tx_desc);
2645         ew32(TDBAL, (tdba & DMA_BIT_MASK(32)));
2646         ew32(TDBAH, (tdba >> 32));
2647         ew32(TDLEN, tdlen);
2648         ew32(TDH, 0);
2649         ew32(TDT, 0);
2650         tx_ring->head = E1000_TDH;
2651         tx_ring->tail = E1000_TDT;
2652
2653         /* Set the default values for the Tx Inter Packet Gap timer */
2654         tipg = DEFAULT_82543_TIPG_IPGT_COPPER;          /*  8  */
2655         ipgr1 = DEFAULT_82543_TIPG_IPGR1;               /*  8  */
2656         ipgr2 = DEFAULT_82543_TIPG_IPGR2;               /*  6  */
2657
2658         if (adapter->flags & FLAG_TIPG_MEDIUM_FOR_80003ESLAN)
2659                 ipgr2 = DEFAULT_80003ES2LAN_TIPG_IPGR2; /*  7  */
2660
2661         tipg |= ipgr1 << E1000_TIPG_IPGR1_SHIFT;
2662         tipg |= ipgr2 << E1000_TIPG_IPGR2_SHIFT;
2663         ew32(TIPG, tipg);
2664
2665         /* Set the Tx Interrupt Delay register */
2666         ew32(TIDV, adapter->tx_int_delay);
2667         /* Tx irq moderation */
2668         ew32(TADV, adapter->tx_abs_int_delay);
2669
2670         if (adapter->flags2 & FLAG2_DMA_BURST) {
2671                 u32 txdctl = er32(TXDCTL(0));
2672                 txdctl &= ~(E1000_TXDCTL_PTHRESH | E1000_TXDCTL_HTHRESH |
2673                             E1000_TXDCTL_WTHRESH);
2674                 /*
2675                  * set up some performance related parameters to encourage the
2676                  * hardware to use the bus more efficiently in bursts, depends
2677                  * on the tx_int_delay to be enabled,
2678                  * wthresh = 5 ==> burst write a cacheline (64 bytes) at a time
2679                  * hthresh = 1 ==> prefetch when one or more available
2680                  * pthresh = 0x1f ==> prefetch if internal cache 31 or less
2681                  * BEWARE: this seems to work but should be considered first if
2682                  * there are Tx hangs or other Tx related bugs
2683                  */
2684                 txdctl |= E1000_TXDCTL_DMA_BURST_ENABLE;
2685                 ew32(TXDCTL(0), txdctl);
2686                 /* erratum work around: set txdctl the same for both queues */
2687                 ew32(TXDCTL(1), txdctl);
2688         }
2689
2690         /* Program the Transmit Control Register */
2691         tctl = er32(TCTL);
2692         tctl &= ~E1000_TCTL_CT;
2693         tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2694                 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2695
2696         if (adapter->flags & FLAG_TARC_SPEED_MODE_BIT) {
2697                 tarc = er32(TARC(0));
2698                 /*
2699                  * set the speed mode bit, we'll clear it if we're not at
2700                  * gigabit link later
2701                  */
2702 #define SPEED_MODE_BIT (1 << 21)
2703                 tarc |= SPEED_MODE_BIT;
2704                 ew32(TARC(0), tarc);
2705         }
2706
2707         /* errata: program both queues to unweighted RR */
2708         if (adapter->flags & FLAG_TARC_SET_BIT_ZERO) {
2709                 tarc = er32(TARC(0));
2710                 tarc |= 1;
2711                 ew32(TARC(0), tarc);
2712                 tarc = er32(TARC(1));
2713                 tarc |= 1;
2714                 ew32(TARC(1), tarc);
2715         }
2716
2717         /* Setup Transmit Descriptor Settings for eop descriptor */
2718         adapter->txd_cmd = E1000_TXD_CMD_EOP | E1000_TXD_CMD_IFCS;
2719
2720         /* only set IDE if we are delaying interrupts using the timers */
2721         if (adapter->tx_int_delay)
2722                 adapter->txd_cmd |= E1000_TXD_CMD_IDE;
2723
2724         /* enable Report Status bit */
2725         adapter->txd_cmd |= E1000_TXD_CMD_RS;
2726
2727         ew32(TCTL, tctl);
2728
2729         e1000e_config_collision_dist(hw);
2730 }
2731
2732 /**
2733  * e1000_setup_rctl - configure the receive control registers
2734  * @adapter: Board private structure
2735  **/
2736 #define PAGE_USE_COUNT(S) (((S) >> PAGE_SHIFT) + \
2737                            (((S) & (PAGE_SIZE - 1)) ? 1 : 0))
2738 static void e1000_setup_rctl(struct e1000_adapter *adapter)
2739 {
2740         struct e1000_hw *hw = &adapter->hw;
2741         u32 rctl, rfctl;
2742         u32 pages = 0;
2743
2744         /* Workaround Si errata on 82579 - configure jumbo frame flow */
2745         if (hw->mac.type == e1000_pch2lan) {
2746                 s32 ret_val;
2747
2748                 if (adapter->netdev->mtu > ETH_DATA_LEN)
2749                         ret_val = e1000_lv_jumbo_workaround_ich8lan(hw, true);
2750                 else
2751                         ret_val = e1000_lv_jumbo_workaround_ich8lan(hw, false);
2752
2753                 if (ret_val)
2754                         e_dbg("failed to enable jumbo frame workaround mode\n");
2755         }
2756
2757         /* Program MC offset vector base */
2758         rctl = er32(RCTL);
2759         rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
2760         rctl |= E1000_RCTL_EN | E1000_RCTL_BAM |
2761                 E1000_RCTL_LBM_NO | E1000_RCTL_RDMTS_HALF |
2762                 (adapter->hw.mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
2763
2764         /* Do not Store bad packets */
2765         rctl &= ~E1000_RCTL_SBP;
2766
2767         /* Enable Long Packet receive */
2768         if (adapter->netdev->mtu <= ETH_DATA_LEN)
2769                 rctl &= ~E1000_RCTL_LPE;
2770         else
2771                 rctl |= E1000_RCTL_LPE;
2772
2773         /* Some systems expect that the CRC is included in SMBUS traffic. The
2774          * hardware strips the CRC before sending to both SMBUS (BMC) and to
2775          * host memory when this is enabled
2776          */
2777         if (adapter->flags2 & FLAG2_CRC_STRIPPING)
2778                 rctl |= E1000_RCTL_SECRC;
2779
2780         /* Workaround Si errata on 82577 PHY - configure IPG for jumbos */
2781         if ((hw->phy.type == e1000_phy_82577) && (rctl & E1000_RCTL_LPE)) {
2782                 u16 phy_data;
2783
2784                 e1e_rphy(hw, PHY_REG(770, 26), &phy_data);
2785                 phy_data &= 0xfff8;
2786                 phy_data |= (1 << 2);
2787                 e1e_wphy(hw, PHY_REG(770, 26), phy_data);
2788
2789                 e1e_rphy(hw, 22, &phy_data);
2790                 phy_data &= 0x0fff;
2791                 phy_data |= (1 << 14);
2792                 e1e_wphy(hw, 0x10, 0x2823);
2793                 e1e_wphy(hw, 0x11, 0x0003);
2794                 e1e_wphy(hw, 22, phy_data);
2795         }
2796
2797         /* Setup buffer sizes */
2798         rctl &= ~E1000_RCTL_SZ_4096;
2799         rctl |= E1000_RCTL_BSEX;
2800         switch (adapter->rx_buffer_len) {
2801         case 2048:
2802         default:
2803                 rctl |= E1000_RCTL_SZ_2048;
2804                 rctl &= ~E1000_RCTL_BSEX;
2805                 break;
2806         case 4096:
2807                 rctl |= E1000_RCTL_SZ_4096;
2808                 break;
2809         case 8192:
2810                 rctl |= E1000_RCTL_SZ_8192;
2811                 break;
2812         case 16384:
2813                 rctl |= E1000_RCTL_SZ_16384;
2814                 break;
2815         }
2816
2817         /*
2818          * 82571 and greater support packet-split where the protocol
2819          * header is placed in skb->data and the packet data is
2820          * placed in pages hanging off of skb_shinfo(skb)->nr_frags.
2821          * In the case of a non-split, skb->data is linearly filled,
2822          * followed by the page buffers.  Therefore, skb->data is
2823          * sized to hold the largest protocol header.
2824          *
2825          * allocations using alloc_page take too long for regular MTU
2826          * so only enable packet split for jumbo frames
2827          *
2828          * Using pages when the page size is greater than 16k wastes
2829          * a lot of memory, since we allocate 3 pages at all times
2830          * per packet.
2831          */
2832         pages = PAGE_USE_COUNT(adapter->netdev->mtu);
2833         if (!(adapter->flags & FLAG_HAS_ERT) && (pages <= 3) &&
2834             (PAGE_SIZE <= 16384) && (rctl & E1000_RCTL_LPE))
2835                 adapter->rx_ps_pages = pages;
2836         else
2837                 adapter->rx_ps_pages = 0;
2838
2839         if (adapter->rx_ps_pages) {
2840                 u32 psrctl = 0;
2841
2842                 /* Configure extra packet-split registers */
2843                 rfctl = er32(RFCTL);
2844                 rfctl |= E1000_RFCTL_EXTEN;
2845                 /*
2846                  * disable packet split support for IPv6 extension headers,
2847                  * because some malformed IPv6 headers can hang the Rx
2848                  */
2849                 rfctl |= (E1000_RFCTL_IPV6_EX_DIS |
2850                           E1000_RFCTL_NEW_IPV6_EXT_DIS);
2851
2852                 ew32(RFCTL, rfctl);
2853
2854                 /* Enable Packet split descriptors */
2855                 rctl |= E1000_RCTL_DTYP_PS;
2856
2857                 psrctl |= adapter->rx_ps_bsize0 >>
2858                         E1000_PSRCTL_BSIZE0_SHIFT;
2859
2860                 switch (adapter->rx_ps_pages) {
2861                 case 3:
2862                         psrctl |= PAGE_SIZE <<
2863                                 E1000_PSRCTL_BSIZE3_SHIFT;
2864                 case 2:
2865                         psrctl |= PAGE_SIZE <<
2866                                 E1000_PSRCTL_BSIZE2_SHIFT;
2867                 case 1:
2868                         psrctl |= PAGE_SIZE >>
2869                                 E1000_PSRCTL_BSIZE1_SHIFT;
2870                         break;
2871                 }
2872
2873                 ew32(PSRCTL, psrctl);
2874         }
2875
2876         ew32(RCTL, rctl);
2877         /* just started the receive unit, no need to restart */
2878         adapter->flags &= ~FLAG_RX_RESTART_NOW;
2879 }
2880
2881 /**
2882  * e1000_configure_rx - Configure Receive Unit after Reset
2883  * @adapter: board private structure
2884  *
2885  * Configure the Rx unit of the MAC after a reset.
2886  **/
2887 static void e1000_configure_rx(struct e1000_adapter *adapter)
2888 {
2889         struct e1000_hw *hw = &adapter->hw;
2890         struct e1000_ring *rx_ring = adapter->rx_ring;
2891         u64 rdba;
2892         u32 rdlen, rctl, rxcsum, ctrl_ext;
2893
2894         if (adapter->rx_ps_pages) {
2895                 /* this is a 32 byte descriptor */
2896                 rdlen = rx_ring->count *
2897                     sizeof(union e1000_rx_desc_packet_split);
2898                 adapter->clean_rx = e1000_clean_rx_irq_ps;
2899                 adapter->alloc_rx_buf = e1000_alloc_rx_buffers_ps;
2900         } else if (adapter->netdev->mtu > ETH_FRAME_LEN + ETH_FCS_LEN) {
2901                 rdlen = rx_ring->count * sizeof(struct e1000_rx_desc);
2902                 adapter->clean_rx = e1000_clean_jumbo_rx_irq;
2903                 adapter->alloc_rx_buf = e1000_alloc_jumbo_rx_buffers;
2904         } else {
2905                 rdlen = rx_ring->count * sizeof(struct e1000_rx_desc);
2906                 adapter->clean_rx = e1000_clean_rx_irq;
2907                 adapter->alloc_rx_buf = e1000_alloc_rx_buffers;
2908         }
2909
2910         /* disable receives while setting up the descriptors */
2911         rctl = er32(RCTL);
2912         ew32(RCTL, rctl & ~E1000_RCTL_EN);
2913         e1e_flush();
2914         usleep_range(10000, 20000);
2915
2916         if (adapter->flags2 & FLAG2_DMA_BURST) {
2917                 /*
2918                  * set the writeback threshold (only takes effect if the RDTR
2919                  * is set). set GRAN=1 and write back up to 0x4 worth, and
2920                  * enable prefetching of 0x20 Rx descriptors
2921                  * granularity = 01
2922                  * wthresh = 04,
2923                  * hthresh = 04,
2924                  * pthresh = 0x20
2925                  */
2926                 ew32(RXDCTL(0), E1000_RXDCTL_DMA_BURST_ENABLE);
2927                 ew32(RXDCTL(1), E1000_RXDCTL_DMA_BURST_ENABLE);
2928
2929                 /*
2930                  * override the delay timers for enabling bursting, only if
2931                  * the value was not set by the user via module options
2932                  */
2933                 if (adapter->rx_int_delay == DEFAULT_RDTR)
2934                         adapter->rx_int_delay = BURST_RDTR;
2935                 if (adapter->rx_abs_int_delay == DEFAULT_RADV)
2936                         adapter->rx_abs_int_delay = BURST_RADV;
2937         }
2938
2939         /* set the Receive Delay Timer Register */
2940         ew32(RDTR, adapter->rx_int_delay);
2941
2942         /* irq moderation */
2943         ew32(RADV, adapter->rx_abs_int_delay);
2944         if ((adapter->itr_setting != 0) && (adapter->itr != 0))
2945                 ew32(ITR, 1000000000 / (adapter->itr * 256));
2946
2947         ctrl_ext = er32(CTRL_EXT);
2948         /* Auto-Mask interrupts upon ICR access */
2949         ctrl_ext |= E1000_CTRL_EXT_IAME;
2950         ew32(IAM, 0xffffffff);
2951         ew32(CTRL_EXT, ctrl_ext);
2952         e1e_flush();
2953
2954         /*
2955          * Setup the HW Rx Head and Tail Descriptor Pointers and
2956          * the Base and Length of the Rx Descriptor Ring
2957          */
2958         rdba = rx_ring->dma;
2959         ew32(RDBAL, (rdba & DMA_BIT_MASK(32)));
2960         ew32(RDBAH, (rdba >> 32));
2961         ew32(RDLEN, rdlen);
2962         ew32(RDH, 0);
2963         ew32(RDT, 0);
2964         rx_ring->head = E1000_RDH;
2965         rx_ring->tail = E1000_RDT;
2966
2967         /* Enable Receive Checksum Offload for TCP and UDP */
2968         rxcsum = er32(RXCSUM);
2969         if (adapter->flags & FLAG_RX_CSUM_ENABLED) {
2970                 rxcsum |= E1000_RXCSUM_TUOFL;
2971
2972                 /*
2973                  * IPv4 payload checksum for UDP fragments must be
2974                  * used in conjunction with packet-split.
2975                  */
2976                 if (adapter->rx_ps_pages)
2977                         rxcsum |= E1000_RXCSUM_IPPCSE;
2978         } else {
2979                 rxcsum &= ~E1000_RXCSUM_TUOFL;
2980                 /* no need to clear IPPCSE as it defaults to 0 */
2981         }
2982         ew32(RXCSUM, rxcsum);
2983
2984         /*
2985          * Enable early receives on supported devices, only takes effect when
2986          * packet size is equal or larger than the specified value (in 8 byte
2987          * units), e.g. using jumbo frames when setting to E1000_ERT_2048
2988          */
2989         if ((adapter->flags & FLAG_HAS_ERT) ||
2990             (adapter->hw.mac.type == e1000_pch2lan)) {
2991                 if (adapter->netdev->mtu > ETH_DATA_LEN) {
2992                         u32 rxdctl = er32(RXDCTL(0));
2993                         ew32(RXDCTL(0), rxdctl | 0x3);
2994                         if (adapter->flags & FLAG_HAS_ERT)
2995                                 ew32(ERT, E1000_ERT_2048 | (1 << 13));
2996                         /*
2997                          * With jumbo frames and early-receive enabled,
2998                          * excessive C-state transition latencies result in
2999                          * dropped transactions.
3000                          */
3001                         pm_qos_update_request(&adapter->netdev->pm_qos_req, 55);
3002                 } else {
3003                         pm_qos_update_request(&adapter->netdev->pm_qos_req,
3004                                               PM_QOS_DEFAULT_VALUE);
3005                 }
3006         }
3007
3008         /* Enable Receives */
3009         ew32(RCTL, rctl);
3010 }
3011
3012 /**
3013  *  e1000_update_mc_addr_list - Update Multicast addresses
3014  *  @hw: pointer to the HW structure
3015  *  @mc_addr_list: array of multicast addresses to program
3016  *  @mc_addr_count: number of multicast addresses to program
3017  *
3018  *  Updates the Multicast Table Array.
3019  *  The caller must have a packed mc_addr_list of multicast addresses.
3020  **/
3021 static void e1000_update_mc_addr_list(struct e1000_hw *hw, u8 *mc_addr_list,
3022                                       u32 mc_addr_count)
3023 {
3024         hw->mac.ops.update_mc_addr_list(hw, mc_addr_list, mc_addr_count);
3025 }
3026
3027 /**
3028  * e1000_set_multi - Multicast and Promiscuous mode set
3029  * @netdev: network interface device structure
3030  *
3031  * The set_multi entry point is called whenever the multicast address
3032  * list or the network interface flags are updated.  This routine is
3033  * responsible for configuring the hardware for proper multicast,
3034  * promiscuous mode, and all-multi behavior.
3035  **/
3036 static void e1000_set_multi(struct net_device *netdev)
3037 {
3038         struct e1000_adapter *adapter = netdev_priv(netdev);
3039         struct e1000_hw *hw = &adapter->hw;
3040         struct netdev_hw_addr *ha;
3041         u8  *mta_list;
3042         u32 rctl;
3043
3044         /* Check for Promiscuous and All Multicast modes */
3045
3046         rctl = er32(RCTL);
3047
3048         if (netdev->flags & IFF_PROMISC) {
3049                 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
3050                 rctl &= ~E1000_RCTL_VFE;
3051                 /* Do not hardware filter VLANs in promisc mode */
3052                 e1000e_vlan_filter_disable(adapter);
3053         } else {
3054                 if (netdev->flags & IFF_ALLMULTI) {
3055                         rctl |= E1000_RCTL_MPE;
3056                         rctl &= ~E1000_RCTL_UPE;
3057                 } else {
3058                         rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE);
3059                 }
3060                 e1000e_vlan_filter_enable(adapter);
3061         }
3062
3063         ew32(RCTL, rctl);
3064
3065         if (!netdev_mc_empty(netdev)) {
3066                 int i = 0;
3067
3068                 mta_list = kmalloc(netdev_mc_count(netdev) * 6, GFP_ATOMIC);
3069                 if (!mta_list)
3070                         return;
3071
3072                 /* prepare a packed array of only addresses. */
3073                 netdev_for_each_mc_addr(ha, netdev)
3074                         memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
3075
3076                 e1000_update_mc_addr_list(hw, mta_list, i);
3077                 kfree(mta_list);
3078         } else {
3079                 /*
3080                  * if we're called from probe, we might not have
3081                  * anything to do here, so clear out the list
3082                  */
3083                 e1000_update_mc_addr_list(hw, NULL, 0);
3084         }
3085
3086         if (netdev->features & NETIF_F_HW_VLAN_RX)
3087                 e1000e_vlan_strip_enable(adapter);
3088         else
3089                 e1000e_vlan_strip_disable(adapter);
3090 }
3091
3092 /**
3093  * e1000_configure - configure the hardware for Rx and Tx
3094  * @adapter: private board structure
3095  **/
3096 static void e1000_configure(struct e1000_adapter *adapter)
3097 {
3098         e1000_set_multi(adapter->netdev);
3099
3100         e1000_restore_vlan(adapter);
3101         e1000_init_manageability_pt(adapter);
3102
3103         e1000_configure_tx(adapter);
3104         e1000_setup_rctl(adapter);
3105         e1000_configure_rx(adapter);
3106         adapter->alloc_rx_buf(adapter, e1000_desc_unused(adapter->rx_ring));
3107 }
3108
3109 /**
3110  * e1000e_power_up_phy - restore link in case the phy was powered down
3111  * @adapter: address of board private structure
3112  *
3113  * The phy may be powered down to save power and turn off link when the
3114  * driver is unloaded and wake on lan is not enabled (among others)
3115  * *** this routine MUST be followed by a call to e1000e_reset ***
3116  **/
3117 void e1000e_power_up_phy(struct e1000_adapter *adapter)
3118 {
3119         if (adapter->hw.phy.ops.power_up)
3120                 adapter->hw.phy.ops.power_up(&adapter->hw);
3121
3122         adapter->hw.mac.ops.setup_link(&adapter->hw);
3123 }
3124
3125 /**
3126  * e1000_power_down_phy - Power down the PHY
3127  *
3128  * Power down the PHY so no link is implied when interface is down.
3129  * The PHY cannot be powered down if management or WoL is active.
3130  */
3131 static void e1000_power_down_phy(struct e1000_adapter *adapter)
3132 {
3133         /* WoL is enabled */
3134         if (adapter->wol)
3135                 return;
3136
3137         if (adapter->hw.phy.ops.power_down)
3138                 adapter->hw.phy.ops.power_down(&adapter->hw);
3139 }
3140
3141 /**
3142  * e1000e_reset - bring the hardware into a known good state
3143  *
3144  * This function boots the hardware and enables some settings that
3145  * require a configuration cycle of the hardware - those cannot be
3146  * set/changed during runtime. After reset the device needs to be
3147  * properly configured for Rx, Tx etc.
3148  */
3149 void e1000e_reset(struct e1000_adapter *adapter)
3150 {
3151         struct e1000_mac_info *mac = &adapter->hw.mac;
3152         struct e1000_fc_info *fc = &adapter->hw.fc;
3153         struct e1000_hw *hw = &adapter->hw;
3154         u32 tx_space, min_tx_space, min_rx_space;
3155         u32 pba = adapter->pba;
3156         u16 hwm;
3157
3158         /* reset Packet Buffer Allocation to default */
3159         ew32(PBA, pba);
3160
3161         if (adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) {
3162                 /*
3163                  * To maintain wire speed transmits, the Tx FIFO should be
3164                  * large enough to accommodate two full transmit packets,
3165                  * rounded up to the next 1KB and expressed in KB.  Likewise,
3166                  * the Rx FIFO should be large enough to accommodate at least
3167                  * one full receive packet and is similarly rounded up and
3168                  * expressed in KB.
3169                  */
3170                 pba = er32(PBA);
3171                 /* upper 16 bits has Tx packet buffer allocation size in KB */
3172                 tx_space = pba >> 16;
3173                 /* lower 16 bits has Rx packet buffer allocation size in KB */
3174                 pba &= 0xffff;
3175                 /*
3176                  * the Tx fifo also stores 16 bytes of information about the Tx
3177                  * but don't include ethernet FCS because hardware appends it
3178                  */
3179                 min_tx_space = (adapter->max_frame_size +
3180                                 sizeof(struct e1000_tx_desc) -
3181                                 ETH_FCS_LEN) * 2;
3182                 min_tx_space = ALIGN(min_tx_space, 1024);
3183                 min_tx_space >>= 10;
3184                 /* software strips receive CRC, so leave room for it */
3185                 min_rx_space = adapter->max_frame_size;
3186                 min_rx_space = ALIGN(min_rx_space, 1024);
3187                 min_rx_space >>= 10;
3188
3189                 /*
3190                  * If current Tx allocation is less than the min Tx FIFO size,
3191                  * and the min Tx FIFO size is less than the current Rx FIFO
3192                  * allocation, take space away from current Rx allocation
3193                  */
3194                 if ((tx_space < min_tx_space) &&
3195                     ((min_tx_space - tx_space) < pba)) {
3196                         pba -= min_tx_space - tx_space;
3197
3198                         /*
3199                          * if short on Rx space, Rx wins and must trump Tx
3200                          * adjustment or use Early Receive if available
3201                          */
3202                         if ((pba < min_rx_space) &&
3203                             (!(adapter->flags & FLAG_HAS_ERT)))
3204                                 /* ERT enabled in e1000_configure_rx */
3205                                 pba = min_rx_space;
3206                 }
3207
3208                 ew32(PBA, pba);
3209         }
3210
3211         /*
3212          * flow control settings
3213          *
3214          * The high water mark must be low enough to fit one full frame
3215          * (or the size used for early receive) above it in the Rx FIFO.
3216          * Set it to the lower of:
3217          * - 90% of the Rx FIFO size, and
3218          * - the full Rx FIFO size minus the early receive size (for parts
3219          *   with ERT support assuming ERT set to E1000_ERT_2048), or
3220          * - the full Rx FIFO size minus one full frame
3221          */
3222         if (adapter->flags & FLAG_DISABLE_FC_PAUSE_TIME)
3223                 fc->pause_time = 0xFFFF;
3224         else
3225                 fc->pause_time = E1000_FC_PAUSE_TIME;
3226         fc->send_xon = 1;
3227         fc->current_mode = fc->requested_mode;
3228
3229         switch (hw->mac.type) {
3230         default:
3231                 if ((adapter->flags & FLAG_HAS_ERT) &&
3232                     (adapter->netdev->mtu > ETH_DATA_LEN))
3233                         hwm = min(((pba << 10) * 9 / 10),
3234                                   ((pba << 10) - (E1000_ERT_2048 << 3)));
3235                 else
3236                         hwm = min(((pba << 10) * 9 / 10),
3237                                   ((pba << 10) - adapter->max_frame_size));
3238
3239                 fc->high_water = hwm & E1000_FCRTH_RTH; /* 8-byte granularity */
3240                 fc->low_water = fc->high_water - 8;
3241                 break;
3242         case e1000_pchlan:
3243                 /*
3244                  * Workaround PCH LOM adapter hangs with certain network
3245                  * loads.  If hangs persist, try disabling Tx flow control.
3246                  */
3247                 if (adapter->netdev->mtu > ETH_DATA_LEN) {
3248                         fc->high_water = 0x3500;
3249                         fc->low_water  = 0x1500;
3250                 } else {
3251                         fc->high_water = 0x5000;
3252                         fc->low_water  = 0x3000;
3253                 }
3254                 fc->refresh_time = 0x1000;
3255                 break;
3256         case e1000_pch2lan:
3257                 fc->high_water = 0x05C20;
3258                 fc->low_water = 0x05048;
3259                 fc->pause_time = 0x0650;
3260                 fc->refresh_time = 0x0400;
3261                 if (adapter->netdev->mtu > ETH_DATA_LEN) {
3262                         pba = 14;
3263                         ew32(PBA, pba);
3264                 }
3265                 break;
3266         }
3267
3268         /*
3269          * Disable Adaptive Interrupt Moderation if 2 full packets cannot
3270          * fit in receive buffer and early-receive not supported.
3271          */
3272         if (adapter->itr_setting & 0x3) {
3273                 if (((adapter->max_frame_size * 2) > (pba << 10)) &&
3274                     !(adapter->flags & FLAG_HAS_ERT)) {
3275                         if (!(adapter->flags2 & FLAG2_DISABLE_AIM)) {
3276                                 dev_info(&adapter->pdev->dev,
3277                                         "Interrupt Throttle Rate turned off\n");
3278                                 adapter->flags2 |= FLAG2_DISABLE_AIM;
3279                                 ew32(ITR, 0);
3280                         }
3281                 } else if (adapter->flags2 & FLAG2_DISABLE_AIM) {
3282                         dev_info(&adapter->pdev->dev,
3283                                  "Interrupt Throttle Rate turned on\n");
3284                         adapter->flags2 &= ~FLAG2_DISABLE_AIM;
3285                         adapter->itr = 20000;
3286                         ew32(ITR, 1000000000 / (adapter->itr * 256));
3287                 }
3288         }
3289
3290         /* Allow time for pending master requests to run */
3291         mac->ops.reset_hw(hw);
3292
3293         /*
3294          * For parts with AMT enabled, let the firmware know
3295          * that the network interface is in control
3296          */
3297         if (adapter->flags & FLAG_HAS_AMT)
3298                 e1000e_get_hw_control(adapter);
3299
3300         ew32(WUC, 0);
3301
3302         if (mac->ops.init_hw(hw))
3303                 e_err("Hardware Error\n");
3304
3305         e1000_update_mng_vlan(adapter);
3306
3307         /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
3308         ew32(VET, ETH_P_8021Q);
3309
3310         e1000e_reset_adaptive(hw);
3311
3312         if (!netif_running(adapter->netdev) &&
3313             !test_bit(__E1000_TESTING, &adapter->state)) {
3314                 e1000_power_down_phy(adapter);
3315                 return;
3316         }
3317
3318         e1000_get_phy_info(hw);
3319
3320         if ((adapter->flags & FLAG_HAS_SMART_POWER_DOWN) &&
3321             !(adapter->flags & FLAG_SMART_POWER_DOWN)) {
3322                 u16 phy_data = 0;
3323                 /*
3324                  * speed up time to link by disabling smart power down, ignore
3325                  * the return value of this function because there is nothing
3326                  * different we would do if it failed
3327                  */
3328                 e1e_rphy(hw, IGP02E1000_PHY_POWER_MGMT, &phy_data);
3329                 phy_data &= ~IGP02E1000_PM_SPD;
3330                 e1e_wphy(hw, IGP02E1000_PHY_POWER_MGMT, phy_data);
3331         }
3332 }
3333
3334 int e1000e_up(struct e1000_adapter *adapter)
3335 {
3336         struct e1000_hw *hw = &adapter->hw;
3337
3338         /* hardware has been reset, we need to reload some things */
3339         e1000_configure(adapter);
3340
3341         clear_bit(__E1000_DOWN, &adapter->state);
3342
3343         napi_enable(&adapter->napi);
3344         if (adapter->msix_entries)
3345                 e1000_configure_msix(adapter);
3346         e1000_irq_enable(adapter);
3347
3348         netif_wake_queue(adapter->netdev);
3349
3350         /* fire a link change interrupt to start the watchdog */
3351         if (adapter->msix_entries)
3352                 ew32(ICS, E1000_ICS_LSC | E1000_ICR_OTHER);
3353         else
3354                 ew32(ICS, E1000_ICS_LSC);
3355
3356         return 0;
3357 }
3358
3359 static void e1000e_flush_descriptors(struct e1000_adapter *adapter)
3360 {
3361         struct e1000_hw *hw = &adapter->hw;
3362
3363         if (!(adapter->flags2 & FLAG2_DMA_BURST))
3364                 return;
3365
3366         /* flush pending descriptor writebacks to memory */
3367         ew32(TIDV, adapter->tx_int_delay | E1000_TIDV_FPD);
3368         ew32(RDTR, adapter->rx_int_delay | E1000_RDTR_FPD);
3369
3370         /* execute the writes immediately */
3371         e1e_flush();
3372 }
3373
3374 static void e1000e_update_stats(struct e1000_adapter *adapter);
3375
3376 void e1000e_down(struct e1000_adapter *adapter)
3377 {
3378         struct net_device *netdev = adapter->netdev;
3379         struct e1000_hw *hw = &adapter->hw;
3380         u32 tctl, rctl;
3381
3382         /*
3383          * signal that we're down so the interrupt handler does not
3384          * reschedule our watchdog timer
3385          */
3386         set_bit(__E1000_DOWN, &adapter->state);
3387
3388         /* disable receives in the hardware */
3389         rctl = er32(RCTL);
3390         ew32(RCTL, rctl & ~E1000_RCTL_EN);
3391         /* flush and sleep below */
3392
3393         netif_stop_queue(netdev);
3394
3395         /* disable transmits in the hardware */
3396         tctl = er32(TCTL);
3397         tctl &= ~E1000_TCTL_EN;
3398         ew32(TCTL, tctl);
3399         /* flush both disables and wait for them to finish */
3400         e1e_flush();
3401         usleep_range(10000, 20000);
3402
3403         napi_disable(&adapter->napi);
3404         e1000_irq_disable(adapter);
3405
3406         del_timer_sync(&adapter->watchdog_timer);
3407         del_timer_sync(&adapter->phy_info_timer);
3408
3409         netif_carrier_off(netdev);
3410
3411         spin_lock(&adapter->stats64_lock);
3412         e1000e_update_stats(adapter);
3413         spin_unlock(&adapter->stats64_lock);
3414
3415         adapter->link_speed = 0;
3416         adapter->link_duplex = 0;
3417
3418         if (!pci_channel_offline(adapter->pdev))
3419                 e1000e_reset(adapter);
3420
3421         e1000e_flush_descriptors(adapter);
3422
3423         e1000_clean_tx_ring(adapter);
3424         e1000_clean_rx_ring(adapter);
3425
3426         /*
3427          * TODO: for power management, we could drop the link and
3428          * pci_disable_device here.
3429          */
3430 }
3431
3432 void e1000e_reinit_locked(struct e1000_adapter *adapter)
3433 {
3434         might_sleep();
3435         while (test_and_set_bit(__E1000_RESETTING, &adapter->state))
3436                 usleep_range(1000, 2000);
3437         e1000e_down(adapter);
3438         e1000e_up(adapter);
3439         clear_bit(__E1000_RESETTING, &adapter->state);
3440 }
3441
3442 /**
3443  * e1000_sw_init - Initialize general software structures (struct e1000_adapter)
3444  * @adapter: board private structure to initialize
3445  *
3446  * e1000_sw_init initializes the Adapter private data structure.
3447  * Fields are initialized based on PCI device information and
3448  * OS network device settings (MTU size).
3449  **/
3450 static int __devinit e1000_sw_init(struct e1000_adapter *adapter)
3451 {
3452         struct net_device *netdev = adapter->netdev;
3453
3454         adapter->rx_buffer_len = ETH_FRAME_LEN + VLAN_HLEN + ETH_FCS_LEN;
3455         adapter->rx_ps_bsize0 = 128;
3456         adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
3457         adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
3458
3459         spin_lock_init(&adapter->stats64_lock);
3460
3461         e1000e_set_interrupt_capability(adapter);
3462
3463         if (e1000_alloc_queues(adapter))
3464                 return -ENOMEM;
3465
3466         /* Explicitly disable IRQ since the NIC can be in any state. */
3467         e1000_irq_disable(adapter);
3468
3469         set_bit(__E1000_DOWN, &adapter->state);
3470         return 0;
3471 }
3472
3473 /**
3474  * e1000_intr_msi_test - Interrupt Handler
3475  * @irq: interrupt number
3476  * @data: pointer to a network interface device structure
3477  **/
3478 static irqreturn_t e1000_intr_msi_test(int irq, void *data)
3479 {
3480         struct net_device *netdev = data;
3481         struct e1000_adapter *adapter = netdev_priv(netdev);
3482         struct e1000_hw *hw = &adapter->hw;
3483         u32 icr = er32(ICR);
3484
3485         e_dbg("icr is %08X\n", icr);
3486         if (icr & E1000_ICR_RXSEQ) {
3487                 adapter->flags &= ~FLAG_MSI_TEST_FAILED;
3488                 wmb();
3489         }
3490
3491         return IRQ_HANDLED;
3492 }
3493
3494 /**
3495  * e1000_test_msi_interrupt - Returns 0 for successful test
3496  * @adapter: board private struct
3497  *
3498  * code flow taken from tg3.c
3499  **/
3500 static int e1000_test_msi_interrupt(struct e1000_adapter *adapter)
3501 {
3502         struct net_device *netdev = adapter->netdev;
3503         struct e1000_hw *hw = &adapter->hw;
3504         int err;
3505
3506         /* poll_enable hasn't been called yet, so don't need disable */
3507         /* clear any pending events */
3508         er32(ICR);
3509
3510         /* free the real vector and request a test handler */
3511         e1000_free_irq(adapter);
3512         e1000e_reset_interrupt_capability(adapter);
3513
3514         /* Assume that the test fails, if it succeeds then the test
3515          * MSI irq handler will unset this flag */
3516         adapter->flags |= FLAG_MSI_TEST_FAILED;
3517
3518         err = pci_enable_msi(adapter->pdev);
3519         if (err)
3520                 goto msi_test_failed;
3521
3522         err = request_irq(adapter->pdev->irq, e1000_intr_msi_test, 0,
3523                           netdev->name, netdev);
3524         if (err) {
3525                 pci_disable_msi(adapter->pdev);
3526                 goto msi_test_failed;
3527         }
3528
3529         wmb();
3530
3531         e1000_irq_enable(adapter);
3532
3533         /* fire an unusual interrupt on the test handler */
3534         ew32(ICS, E1000_ICS_RXSEQ);
3535         e1e_flush();
3536         msleep(50);
3537
3538         e1000_irq_disable(adapter);
3539
3540         rmb();
3541
3542         if (adapter->flags & FLAG_MSI_TEST_FAILED) {
3543                 adapter->int_mode = E1000E_INT_MODE_LEGACY;
3544                 e_info("MSI interrupt test failed, using legacy interrupt.\n");
3545         } else
3546                 e_dbg("MSI interrupt test succeeded!\n");
3547
3548         free_irq(adapter->pdev->irq, netdev);
3549         pci_disable_msi(adapter->pdev);
3550
3551 msi_test_failed:
3552         e1000e_set_interrupt_capability(adapter);
3553         return e1000_request_irq(adapter);
3554 }
3555
3556 /**
3557  * e1000_test_msi - Returns 0 if MSI test succeeds or INTx mode is restored
3558  * @adapter: board private struct
3559  *
3560  * code flow taken from tg3.c, called with e1000 interrupts disabled.
3561  **/
3562 static int e1000_test_msi(struct e1000_adapter *adapter)
3563 {
3564         int err;
3565         u16 pci_cmd;
3566
3567         if (!(adapter->flags & FLAG_MSI_ENABLED))
3568                 return 0;
3569
3570         /* disable SERR in case the MSI write causes a master abort */
3571         pci_read_config_word(adapter->pdev, PCI_COMMAND, &pci_cmd);
3572         if (pci_cmd & PCI_COMMAND_SERR)
3573                 pci_write_config_word(adapter->pdev, PCI_COMMAND,
3574                                       pci_cmd & ~PCI_COMMAND_SERR);
3575
3576         err = e1000_test_msi_interrupt(adapter);
3577
3578         /* re-enable SERR */
3579         if (pci_cmd & PCI_COMMAND_SERR) {
3580                 pci_read_config_word(adapter->pdev, PCI_COMMAND, &pci_cmd);
3581                 pci_cmd |= PCI_COMMAND_SERR;
3582                 pci_write_config_word(adapter->pdev, PCI_COMMAND, pci_cmd);
3583         }
3584
3585         return err;
3586 }
3587
3588 /**
3589  * e1000_open - Called when a network interface is made active
3590  * @netdev: network interface device structure
3591  *
3592  * Returns 0 on success, negative value on failure
3593  *
3594  * The open entry point is called when a network interface is made
3595  * active by the system (IFF_UP).  At this point all resources needed
3596  * for transmit and receive operations are allocated, the interrupt
3597  * handler is registered with the OS, the watchdog timer is started,
3598  * and the stack is notified that the interface is ready.
3599  **/
3600 static int e1000_open(struct net_device *netdev)
3601 {
3602         struct e1000_adapter *adapter = netdev_priv(netdev);
3603         struct e1000_hw *hw = &adapter->hw;
3604         struct pci_dev *pdev = adapter->pdev;
3605         int err;
3606
3607         /* disallow open during test */
3608         if (test_bit(__E1000_TESTING, &adapter->state))
3609                 return -EBUSY;
3610
3611         pm_runtime_get_sync(&pdev->dev);
3612
3613         netif_carrier_off(netdev);
3614
3615         /* allocate transmit descriptors */
3616         err = e1000e_setup_tx_resources(adapter);
3617         if (err)
3618                 goto err_setup_tx;
3619
3620         /* allocate receive descriptors */
3621         err = e1000e_setup_rx_resources(adapter);
3622         if (err)
3623                 goto err_setup_rx;
3624
3625         /*
3626          * If AMT is enabled, let the firmware know that the network
3627          * interface is now open and reset the part to a known state.
3628          */
3629         if (adapter->flags & FLAG_HAS_AMT) {
3630                 e1000e_get_hw_control(adapter);
3631                 e1000e_reset(adapter);
3632         }
3633
3634         e1000e_power_up_phy(adapter);
3635
3636         adapter->mng_vlan_id = E1000_MNG_VLAN_NONE;
3637         if ((adapter->hw.mng_cookie.status &
3638              E1000_MNG_DHCP_COOKIE_STATUS_VLAN))
3639                 e1000_update_mng_vlan(adapter);
3640
3641         /* DMA latency requirement to workaround early-receive/jumbo issue */
3642         if ((adapter->flags & FLAG_HAS_ERT) ||
3643             (adapter->hw.mac.type == e1000_pch2lan))
3644                 pm_qos_add_request(&adapter->netdev->pm_qos_req,
3645                                    PM_QOS_CPU_DMA_LATENCY,
3646                                    PM_QOS_DEFAULT_VALUE);
3647
3648         /*
3649          * before we allocate an interrupt, we must be ready to handle it.
3650          * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
3651          * as soon as we call pci_request_irq, so we have to setup our
3652          * clean_rx handler before we do so.
3653          */
3654         e1000_configure(adapter);
3655
3656         err = e1000_request_irq(adapter);
3657         if (err)
3658                 goto err_req_irq;
3659
3660         /*
3661          * Work around PCIe errata with MSI interrupts causing some chipsets to
3662          * ignore e1000e MSI messages, which means we need to test our MSI
3663          * interrupt now
3664          */
3665         if (adapter->int_mode != E1000E_INT_MODE_LEGACY) {
3666                 err = e1000_test_msi(adapter);
3667                 if (err) {
3668                         e_err("Interrupt allocation failed\n");
3669                         goto err_req_irq;
3670                 }
3671         }
3672
3673         /* From here on the code is the same as e1000e_up() */
3674         clear_bit(__E1000_DOWN, &adapter->state);
3675
3676         napi_enable(&adapter->napi);
3677
3678         e1000_irq_enable(adapter);
3679
3680         netif_start_queue(netdev);
3681
3682         adapter->idle_check = true;
3683         pm_runtime_put(&pdev->dev);
3684
3685         /* fire a link status change interrupt to start the watchdog */
3686         if (adapter->msix_entries)
3687                 ew32(ICS, E1000_ICS_LSC | E1000_ICR_OTHER);
3688         else
3689                 ew32(ICS, E1000_ICS_LSC);
3690
3691         return 0;
3692
3693 err_req_irq:
3694         e1000e_release_hw_control(adapter);
3695         e1000_power_down_phy(adapter);
3696         e1000e_free_rx_resources(adapter);
3697 err_setup_rx:
3698         e1000e_free_tx_resources(adapter);
3699 err_setup_tx:
3700         e1000e_reset(adapter);
3701         pm_runtime_put_sync(&pdev->dev);
3702
3703         return err;
3704 }
3705
3706 /**
3707  * e1000_close - Disables a network interface
3708  * @netdev: network interface device structure
3709  *
3710  * Returns 0, this is not allowed to fail
3711  *
3712  * The close entry point is called when an interface is de-activated
3713  * by the OS.  The hardware is still under the drivers control, but
3714  * needs to be disabled.  A global MAC reset is issued to stop the
3715  * hardware, and all transmit and receive resources are freed.
3716  **/
3717 static int e1000_close(struct net_device *netdev)
3718 {
3719         struct e1000_adapter *adapter = netdev_priv(netdev);
3720         struct pci_dev *pdev = adapter->pdev;
3721
3722         WARN_ON(test_bit(__E1000_RESETTING, &adapter->state));
3723
3724         pm_runtime_get_sync(&pdev->dev);
3725
3726         if (!test_bit(__E1000_DOWN, &adapter->state)) {
3727                 e1000e_down(adapter);
3728                 e1000_free_irq(adapter);
3729         }
3730         e1000_power_down_phy(adapter);
3731
3732         e1000e_free_tx_resources(adapter);
3733         e1000e_free_rx_resources(adapter);
3734
3735         /*
3736          * kill manageability vlan ID if supported, but not if a vlan with
3737          * the same ID is registered on the host OS (let 8021q kill it)
3738          */
3739         if (adapter->hw.mng_cookie.status &
3740             E1000_MNG_DHCP_COOKIE_STATUS_VLAN)
3741                 e1000_vlan_rx_kill_vid(netdev, adapter->mng_vlan_id);
3742
3743         /*
3744          * If AMT is enabled, let the firmware know that the network
3745          * interface is now closed
3746          */
3747         if ((adapter->flags & FLAG_HAS_AMT) &&
3748             !test_bit(__E1000_TESTING, &adapter->state))
3749                 e1000e_release_hw_control(adapter);
3750
3751         if ((adapter->flags & FLAG_HAS_ERT) ||
3752             (adapter->hw.mac.type == e1000_pch2lan))
3753                 pm_qos_remove_request(&adapter->netdev->pm_qos_req);
3754
3755         pm_runtime_put_sync(&pdev->dev);
3756
3757         return 0;
3758 }
3759 /**
3760  * e1000_set_mac - Change the Ethernet Address of the NIC
3761  * @netdev: network interface device structure
3762  * @p: pointer to an address structure
3763  *
3764  * Returns 0 on success, negative on failure
3765  **/
3766 static int e1000_set_mac(struct net_device *netdev, void *p)
3767 {
3768         struct e1000_adapter *adapter = netdev_priv(netdev);
3769         struct sockaddr *addr = p;
3770
3771         if (!is_valid_ether_addr(addr->sa_data))
3772                 return -EADDRNOTAVAIL;
3773
3774         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
3775         memcpy(adapter->hw.mac.addr, addr->sa_data, netdev->addr_len);
3776
3777         e1000e_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
3778
3779         if (adapter->flags & FLAG_RESET_OVERWRITES_LAA) {
3780                 /* activate the work around */
3781                 e1000e_set_laa_state_82571(&adapter->hw, 1);
3782
3783                 /*
3784                  * Hold a copy of the LAA in RAR[14] This is done so that
3785                  * between the time RAR[0] gets clobbered  and the time it
3786                  * gets fixed (in e1000_watchdog), the actual LAA is in one
3787                  * of the RARs and no incoming packets directed to this port
3788                  * are dropped. Eventually the LAA will be in RAR[0] and
3789                  * RAR[14]
3790                  */
3791                 e1000e_rar_set(&adapter->hw,
3792                               adapter->hw.mac.addr,
3793                               adapter->hw.mac.rar_entry_count - 1);
3794         }
3795
3796         return 0;
3797 }
3798
3799 /**
3800  * e1000e_update_phy_task - work thread to update phy
3801  * @work: pointer to our work struct
3802  *
3803  * this worker thread exists because we must acquire a
3804  * semaphore to read the phy, which we could msleep while
3805  * waiting for it, and we can't msleep in a timer.
3806  **/
3807 static void e1000e_update_phy_task(struct work_struct *work)
3808 {
3809         struct e1000_adapter *adapter = container_of(work,
3810                                         struct e1000_adapter, update_phy_task);
3811
3812         if (test_bit(__E1000_DOWN, &adapter->state))
3813                 return;
3814
3815         e1000_get_phy_info(&adapter->hw);
3816 }
3817
3818 /*
3819  * Need to wait a few seconds after link up to get diagnostic information from
3820  * the phy
3821  */
3822 static void e1000_update_phy_info(unsigned long data)
3823 {
3824         struct e1000_adapter *adapter = (struct e1000_adapter *) data;
3825
3826         if (test_bit(__E1000_DOWN, &adapter->state))
3827                 return;
3828
3829         schedule_work(&adapter->update_phy_task);
3830 }
3831
3832 /**
3833  * e1000e_update_phy_stats - Update the PHY statistics counters
3834  * @adapter: board private structure
3835  **/
3836 static void e1000e_update_phy_stats(struct e1000_adapter *adapter)
3837 {
3838         struct e1000_hw *hw = &adapter->hw;
3839         s32 ret_val;
3840         u16 phy_data;
3841
3842         ret_val = hw->phy.ops.acquire(hw);
3843         if (ret_val)
3844                 return;
3845
3846         hw->phy.addr = 1;
3847
3848 #define HV_PHY_STATS_PAGE       778
3849         /*
3850          * A page set is expensive so check if already on desired page.
3851          * If not, set to the page with the PHY status registers.
3852          */
3853         ret_val = e1000e_read_phy_reg_mdic(hw, IGP01E1000_PHY_PAGE_SELECT,
3854                                            &phy_data);
3855         if (ret_val)
3856                 goto release;
3857         if (phy_data != (HV_PHY_STATS_PAGE << IGP_PAGE_SHIFT)) {
3858                 ret_val = e1000e_write_phy_reg_mdic(hw,
3859                                                     IGP01E1000_PHY_PAGE_SELECT,
3860                                                     (HV_PHY_STATS_PAGE <<
3861                                                      IGP_PAGE_SHIFT));
3862                 if (ret_val)
3863                         goto release;
3864         }
3865
3866         /* Read/clear the upper 16-bit registers and read/accumulate lower */
3867
3868         /* Single Collision Count */
3869         e1000e_read_phy_reg_mdic(hw, HV_SCC_UPPER & MAX_PHY_REG_ADDRESS,
3870                                  &phy_data);
3871         ret_val = e1000e_read_phy_reg_mdic(hw,
3872                                            HV_SCC_LOWER & MAX_PHY_REG_ADDRESS,
3873                                            &phy_data);
3874         if (!ret_val)
3875                 adapter->stats.scc += phy_data;
3876
3877         /* Excessive Collision Count */
3878         e1000e_read_phy_reg_mdic(hw, HV_ECOL_UPPER & MAX_PHY_REG_ADDRESS,
3879                                  &phy_data);
3880         ret_val = e1000e_read_phy_reg_mdic(hw,
3881                                            HV_ECOL_LOWER & MAX_PHY_REG_ADDRESS,
3882                                            &phy_data);
3883         if (!ret_val)
3884                 adapter->stats.ecol += phy_data;
3885
3886         /* Multiple Collision Count */
3887         e1000e_read_phy_reg_mdic(hw, HV_MCC_UPPER & MAX_PHY_REG_ADDRESS,
3888                                  &phy_data);
3889         ret_val = e1000e_read_phy_reg_mdic(hw,
3890                                            HV_MCC_LOWER & MAX_PHY_REG_ADDRESS,
3891                                            &phy_data);
3892         if (!ret_val)
3893                 adapter->stats.mcc += phy_data;
3894
3895         /* Late Collision Count */
3896         e1000e_read_phy_reg_mdic(hw, HV_LATECOL_UPPER & MAX_PHY_REG_ADDRESS,
3897                                  &phy_data);
3898         ret_val = e1000e_read_phy_reg_mdic(hw,
3899                                            HV_LATECOL_LOWER &
3900                                            MAX_PHY_REG_ADDRESS,
3901                                            &phy_data);
3902         if (!ret_val)
3903                 adapter->stats.latecol += phy_data;
3904
3905         /* Collision Count - also used for adaptive IFS */
3906         e1000e_read_phy_reg_mdic(hw, HV_COLC_UPPER & MAX_PHY_REG_ADDRESS,
3907                                  &phy_data);
3908         ret_val = e1000e_read_phy_reg_mdic(hw,
3909                                            HV_COLC_LOWER & MAX_PHY_REG_ADDRESS,
3910                                            &phy_data);
3911         if (!ret_val)
3912                 hw->mac.collision_delta = phy_data;
3913
3914         /* Defer Count */
3915         e1000e_read_phy_reg_mdic(hw, HV_DC_UPPER & MAX_PHY_REG_ADDRESS,
3916                                  &phy_data);
3917         ret_val = e1000e_read_phy_reg_mdic(hw,
3918                                            HV_DC_LOWER & MAX_PHY_REG_ADDRESS,
3919                                            &phy_data);
3920         if (!ret_val)
3921                 adapter->stats.dc += phy_data;
3922
3923         /* Transmit with no CRS */
3924         e1000e_read_phy_reg_mdic(hw, HV_TNCRS_UPPER & MAX_PHY_REG_ADDRESS,
3925                                  &phy_data);
3926         ret_val = e1000e_read_phy_reg_mdic(hw,
3927                                            HV_TNCRS_LOWER & MAX_PHY_REG_ADDRESS,
3928                                            &phy_data);
3929         if (!ret_val)
3930                 adapter->stats.tncrs += phy_data;
3931
3932 release:
3933         hw->phy.ops.release(hw);
3934 }
3935
3936 /**
3937  * e1000e_update_stats - Update the board statistics counters
3938  * @adapter: board private structure
3939  **/
3940 static void e1000e_update_stats(struct e1000_adapter *adapter)
3941 {
3942         struct net_device *netdev = adapter->netdev;
3943         struct e1000_hw *hw = &adapter->hw;
3944         struct pci_dev *pdev = adapter->pdev;
3945
3946         /*
3947          * Prevent stats update while adapter is being reset, or if the pci
3948          * connection is down.
3949          */
3950         if (adapter->link_speed == 0)
3951                 return;
3952         if (pci_channel_offline(pdev))
3953                 return;
3954
3955         adapter->stats.crcerrs += er32(CRCERRS);
3956         adapter->stats.gprc += er32(GPRC);
3957         adapter->stats.gorc += er32(GORCL);
3958         er32(GORCH); /* Clear gorc */
3959         adapter->stats.bprc += er32(BPRC);
3960         adapter->stats.mprc += er32(MPRC);
3961         adapter->stats.roc += er32(ROC);
3962
3963         adapter->stats.mpc += er32(MPC);
3964
3965         /* Half-duplex statistics */
3966         if (adapter->link_duplex == HALF_DUPLEX) {
3967                 if (adapter->flags2 & FLAG2_HAS_PHY_STATS) {
3968                         e1000e_update_phy_stats(adapter);
3969                 } else {
3970                         adapter->stats.scc += er32(SCC);
3971                         adapter->stats.ecol += er32(ECOL);
3972                         adapter->stats.mcc += er32(MCC);
3973                         adapter->stats.latecol += er32(LATECOL);
3974                         adapter->stats.dc += er32(DC);
3975
3976                         hw->mac.collision_delta = er32(COLC);
3977
3978                         if ((hw->mac.type != e1000_82574) &&
3979                             (hw->mac.type != e1000_82583))
3980                                 adapter->stats.tncrs += er32(TNCRS);
3981                 }
3982                 adapter->stats.colc += hw->mac.collision_delta;
3983         }
3984
3985         adapter->stats.xonrxc += er32(XONRXC);
3986         adapter->stats.xontxc += er32(XONTXC);
3987         adapter->stats.xoffrxc += er32(XOFFRXC);
3988         adapter->stats.xofftxc += er32(XOFFTXC);
3989         adapter->stats.gptc += er32(GPTC);
3990         adapter->stats.gotc += er32(GOTCL);
3991         er32(GOTCH); /* Clear gotc */
3992         adapter->stats.rnbc += er32(RNBC);
3993         adapter->stats.ruc += er32(RUC);
3994
3995         adapter->stats.mptc += er32(MPTC);
3996         adapter->stats.bptc += er32(BPTC);
3997
3998         /* used for adaptive IFS */
3999
4000         hw->mac.tx_packet_delta = er32(TPT);
4001         adapter->stats.tpt += hw->mac.tx_packet_delta;
4002
4003         adapter->stats.algnerrc += er32(ALGNERRC);
4004         adapter->stats.rxerrc += er32(RXERRC);
4005         adapter->stats.cexterr += er32(CEXTERR);
4006         adapter->stats.tsctc += er32(TSCTC);
4007         adapter->stats.tsctfc += er32(TSCTFC);
4008
4009         /* Fill out the OS statistics structure */
4010         netdev->stats.multicast = adapter->stats.mprc;
4011         netdev->stats.collisions = adapter->stats.colc;
4012
4013         /* Rx Errors */
4014
4015         /*
4016          * RLEC on some newer hardware can be incorrect so build
4017          * our own version based on RUC and ROC
4018          */
4019         netdev->stats.rx_errors = adapter->stats.rxerrc +
4020                 adapter->stats.crcerrs + adapter->stats.algnerrc +
4021                 adapter->stats.ruc + adapter->stats.roc +
4022                 adapter->stats.cexterr;
4023         netdev->stats.rx_length_errors = adapter->stats.ruc +
4024                                               adapter->stats.roc;
4025         netdev->stats.rx_crc_errors = adapter->stats.crcerrs;
4026         netdev->stats.rx_frame_errors = adapter->stats.algnerrc;
4027         netdev->stats.rx_missed_errors = adapter->stats.mpc;
4028
4029         /* Tx Errors */
4030         netdev->stats.tx_errors = adapter->stats.ecol +
4031                                        adapter->stats.latecol;
4032         netdev->stats.tx_aborted_errors = adapter->stats.ecol;
4033         netdev->stats.tx_window_errors = adapter->stats.latecol;
4034         netdev->stats.tx_carrier_errors = adapter->stats.tncrs;
4035
4036         /* Tx Dropped needs to be maintained elsewhere */
4037
4038         /* Management Stats */
4039         adapter->stats.mgptc += er32(MGTPTC);
4040         adapter->stats.mgprc += er32(MGTPRC);
4041         adapter->stats.mgpdc += er32(MGTPDC);
4042 }
4043
4044 /**
4045  * e1000_phy_read_status - Update the PHY register status snapshot
4046  * @adapter: board private structure
4047  **/
4048 static void e1000_phy_read_status(struct e1000_adapter *adapter)
4049 {
4050         struct e1000_hw *hw = &adapter->hw;
4051         struct e1000_phy_regs *phy = &adapter->phy_regs;
4052
4053         if ((er32(STATUS) & E1000_STATUS_LU) &&
4054             (adapter->hw.phy.media_type == e1000_media_type_copper)) {
4055                 int ret_val;
4056
4057                 ret_val  = e1e_rphy(hw, PHY_CONTROL, &phy->bmcr);
4058                 ret_val |= e1e_rphy(hw, PHY_STATUS, &phy->bmsr);
4059                 ret_val |= e1e_rphy(hw, PHY_AUTONEG_ADV, &phy->advertise);
4060                 ret_val |= e1e_rphy(hw, PHY_LP_ABILITY, &phy->lpa);
4061                 ret_val |= e1e_rphy(hw, PHY_AUTONEG_EXP, &phy->expansion);
4062                 ret_val |= e1e_rphy(hw, PHY_1000T_CTRL, &phy->ctrl1000);
4063                 ret_val |= e1e_rphy(hw, PHY_1000T_STATUS, &phy->stat1000);
4064                 ret_val |= e1e_rphy(hw, PHY_EXT_STATUS, &phy->estatus);
4065                 if (ret_val)
4066                         e_warn("Error reading PHY register\n");
4067         } else {
4068                 /*
4069                  * Do not read PHY registers if link is not up
4070                  * Set values to typical power-on defaults
4071                  */
4072                 phy->bmcr = (BMCR_SPEED1000 | BMCR_ANENABLE | BMCR_FULLDPLX);
4073                 phy->bmsr = (BMSR_100FULL | BMSR_100HALF | BMSR_10FULL |
4074                              BMSR_10HALF | BMSR_ESTATEN | BMSR_ANEGCAPABLE |
4075                              BMSR_ERCAP);
4076                 phy->advertise = (ADVERTISE_PAUSE_ASYM | ADVERTISE_PAUSE_CAP |
4077                                   ADVERTISE_ALL | ADVERTISE_CSMA);
4078                 phy->lpa = 0;
4079                 phy->expansion = EXPANSION_ENABLENPAGE;
4080                 phy->ctrl1000 = ADVERTISE_1000FULL;
4081                 phy->stat1000 = 0;
4082                 phy->estatus = (ESTATUS_1000_TFULL | ESTATUS_1000_THALF);
4083         }
4084 }
4085
4086 static void e1000_print_link_info(struct e1000_adapter *adapter)
4087 {
4088         struct e1000_hw *hw = &adapter->hw;
4089         u32 ctrl = er32(CTRL);
4090
4091         /* Link status message must follow this format for user tools */
4092         printk(KERN_INFO "e1000e: %s NIC Link is Up %d Mbps %s, "
4093                "Flow Control: %s\n",
4094                adapter->netdev->name,
4095                adapter->link_speed,
4096                (adapter->link_duplex == FULL_DUPLEX) ?
4097                "Full Duplex" : "Half Duplex",
4098                ((ctrl & E1000_CTRL_TFCE) && (ctrl & E1000_CTRL_RFCE)) ?
4099                "Rx/Tx" :
4100                ((ctrl & E1000_CTRL_RFCE) ? "Rx" :
4101                 ((ctrl & E1000_CTRL_TFCE) ? "Tx" : "None")));
4102 }
4103
4104 static bool e1000e_has_link(struct e1000_adapter *adapter)
4105 {
4106         struct e1000_hw *hw = &adapter->hw;
4107         bool link_active = 0;
4108         s32 ret_val = 0;
4109
4110         /*
4111          * get_link_status is set on LSC (link status) interrupt or
4112          * Rx sequence error interrupt.  get_link_status will stay
4113          * false until the check_for_link establishes link
4114          * for copper adapters ONLY
4115          */
4116         switch (hw->phy.media_type) {
4117         case e1000_media_type_copper:
4118                 if (hw->mac.get_link_status) {
4119                         ret_val = hw->mac.ops.check_for_link(hw);
4120                         link_active = !hw->mac.get_link_status;
4121                 } else {
4122                         link_active = 1;
4123                 }
4124                 break;
4125         case e1000_media_type_fiber:
4126                 ret_val = hw->mac.ops.check_for_link(hw);
4127                 link_active = !!(er32(STATUS) & E1000_STATUS_LU);
4128                 break;
4129         case e1000_media_type_internal_serdes:
4130                 ret_val = hw->mac.ops.check_for_link(hw);
4131                 link_active = adapter->hw.mac.serdes_has_link;
4132                 break;
4133         default:
4134         case e1000_media_type_unknown:
4135                 break;
4136         }
4137
4138         if ((ret_val == E1000_ERR_PHY) && (hw->phy.type == e1000_phy_igp_3) &&
4139             (er32(CTRL) & E1000_PHY_CTRL_GBE_DISABLE)) {
4140                 /* See e1000_kmrn_lock_loss_workaround_ich8lan() */
4141                 e_info("Gigabit has been disabled, downgrading speed\n");
4142         }
4143
4144         return link_active;
4145 }
4146
4147 static void e1000e_enable_receives(struct e1000_adapter *adapter)
4148 {
4149         /* make sure the receive unit is started */
4150         if ((adapter->flags & FLAG_RX_NEEDS_RESTART) &&
4151             (adapter->flags & FLAG_RX_RESTART_NOW)) {
4152                 struct e1000_hw *hw = &adapter->hw;
4153                 u32 rctl = er32(RCTL);
4154                 ew32(RCTL, rctl | E1000_RCTL_EN);
4155                 adapter->flags &= ~FLAG_RX_RESTART_NOW;
4156         }
4157 }
4158
4159 static void e1000e_check_82574_phy_workaround(struct e1000_adapter *adapter)
4160 {
4161         struct e1000_hw *hw = &adapter->hw;
4162
4163         /*
4164          * With 82574 controllers, PHY needs to be checked periodically
4165          * for hung state and reset, if two calls return true
4166          */
4167         if (e1000_check_phy_82574(hw))
4168                 adapter->phy_hang_count++;
4169         else
4170                 adapter->phy_hang_count = 0;
4171
4172         if (adapter->phy_hang_count > 1) {
4173                 adapter->phy_hang_count = 0;
4174                 schedule_work(&adapter->reset_task);
4175         }
4176 }
4177
4178 /**
4179  * e1000_watchdog - Timer Call-back
4180  * @data: pointer to adapter cast into an unsigned long
4181  **/
4182 static void e1000_watchdog(unsigned long data)
4183 {
4184         struct e1000_adapter *adapter = (struct e1000_adapter *) data;
4185
4186         /* Do the rest outside of interrupt context */
4187         schedule_work(&adapter->watchdog_task);
4188
4189         /* TODO: make this use queue_delayed_work() */
4190 }
4191
4192 static void e1000_watchdog_task(struct work_struct *work)
4193 {
4194         struct e1000_adapter *adapter = container_of(work,
4195                                         struct e1000_adapter, watchdog_task);
4196         struct net_device *netdev = adapter->netdev;
4197         struct e1000_mac_info *mac = &adapter->hw.mac;
4198         struct e1000_phy_info *phy = &adapter->hw.phy;
4199         struct e1000_ring *tx_ring = adapter->tx_ring;
4200         struct e1000_hw *hw = &adapter->hw;
4201         u32 link, tctl;
4202
4203         if (test_bit(__E1000_DOWN, &adapter->state))
4204                 return;
4205
4206         link = e1000e_has_link(adapter);
4207         if ((netif_carrier_ok(netdev)) && link) {
4208                 /* Cancel scheduled suspend requests. */
4209                 pm_runtime_resume(netdev->dev.parent);
4210
4211                 e1000e_enable_receives(adapter);
4212                 goto link_up;
4213         }
4214
4215         if ((e1000e_enable_tx_pkt_filtering(hw)) &&
4216             (adapter->mng_vlan_id != adapter->hw.mng_cookie.vlan_id))
4217                 e1000_update_mng_vlan(adapter);
4218
4219         if (link) {
4220                 if (!netif_carrier_ok(netdev)) {
4221                         bool txb2b = 1;
4222
4223                         /* Cancel scheduled suspend requests. */
4224                         pm_runtime_resume(netdev->dev.parent);
4225
4226                         /* update snapshot of PHY registers on LSC */
4227                         e1000_phy_read_status(adapter);
4228                         mac->ops.get_link_up_info(&adapter->hw,
4229                                                    &adapter->link_speed,
4230                                                    &adapter->link_duplex);
4231                         e1000_print_link_info(adapter);
4232                         /*
4233                          * On supported PHYs, check for duplex mismatch only
4234                          * if link has autonegotiated at 10/100 half
4235                          */
4236                         if ((hw->phy.type == e1000_phy_igp_3 ||
4237                              hw->phy.type == e1000_phy_bm) &&
4238                             (hw->mac.autoneg == true) &&
4239                             (adapter->link_speed == SPEED_10 ||
4240                              adapter->link_speed == SPEED_100) &&
4241                             (adapter->link_duplex == HALF_DUPLEX)) {
4242                                 u16 autoneg_exp;
4243
4244                                 e1e_rphy(hw, PHY_AUTONEG_EXP, &autoneg_exp);
4245
4246                                 if (!(autoneg_exp & NWAY_ER_LP_NWAY_CAPS))
4247                                         e_info("Autonegotiated half duplex but"
4248                                                " link partner cannot autoneg. "
4249                                                " Try forcing full duplex if "
4250                                                "link gets many collisions.\n");
4251                         }
4252
4253                         /* adjust timeout factor according to speed/duplex */
4254                         adapter->tx_timeout_factor = 1;
4255                         switch (adapter->link_speed) {
4256                         case SPEED_10:
4257                                 txb2b = 0;
4258                                 adapter->tx_timeout_factor = 16;
4259                                 break;
4260                         case SPEED_100:
4261                                 txb2b = 0;
4262                                 adapter->tx_timeout_factor = 10;
4263                                 break;
4264                         }
4265
4266                         /*
4267                          * workaround: re-program speed mode bit after
4268                          * link-up event
4269                          */
4270                         if ((adapter->flags & FLAG_TARC_SPEED_MODE_BIT) &&
4271                             !txb2b) {
4272                                 u32 tarc0;
4273                                 tarc0 = er32(TARC(0));
4274                                 tarc0 &= ~SPEED_MODE_BIT;
4275                                 ew32(TARC(0), tarc0);
4276                         }
4277
4278                         /*
4279                          * disable TSO for pcie and 10/100 speeds, to avoid
4280                          * some hardware issues
4281                          */
4282                         if (!(adapter->flags & FLAG_TSO_FORCE)) {
4283                                 switch (adapter->link_speed) {
4284                                 case SPEED_10:
4285                                 case SPEED_100:
4286                                         e_info("10/100 speed: disabling TSO\n");
4287                                         netdev->features &= ~NETIF_F_TSO;
4288                                         netdev->features &= ~NETIF_F_TSO6;
4289                                         break;
4290                                 case SPEED_1000:
4291                                         netdev->features |= NETIF_F_TSO;
4292                                         netdev->features |= NETIF_F_TSO6;
4293                                         break;
4294                                 default:
4295                                         /* oops */
4296                                         break;
4297                                 }
4298                         }
4299
4300                         /*
4301                          * enable transmits in the hardware, need to do this
4302                          * after setting TARC(0)
4303                          */
4304                         tctl = er32(TCTL);
4305                         tctl |= E1000_TCTL_EN;
4306                         ew32(TCTL, tctl);
4307
4308                         /*
4309                          * Perform any post-link-up configuration before
4310                          * reporting link up.
4311                          */
4312                         if (phy->ops.cfg_on_link_up)
4313                                 phy->ops.cfg_on_link_up(hw);
4314
4315                         netif_carrier_on(netdev);
4316
4317                         if (!test_bit(__E1000_DOWN, &adapter->state))
4318                                 mod_timer(&adapter->phy_info_timer,
4319                                           round_jiffies(jiffies + 2 * HZ));
4320                 }
4321         } else {
4322                 if (netif_carrier_ok(netdev)) {
4323                         adapter->link_speed = 0;
4324                         adapter->link_duplex = 0;
4325                         /* Link status message must follow this format */
4326                         printk(KERN_INFO "e1000e: %s NIC Link is Down\n",
4327                                adapter->netdev->name);
4328                         netif_carrier_off(netdev);
4329                         if (!test_bit(__E1000_DOWN, &adapter->state))
4330                                 mod_timer(&adapter->phy_info_timer,
4331                                           round_jiffies(jiffies + 2 * HZ));
4332
4333                         if (adapter->flags & FLAG_RX_NEEDS_RESTART)
4334                                 schedule_work(&adapter->reset_task);
4335                         else
4336                                 pm_schedule_suspend(netdev->dev.parent,
4337                                                         LINK_TIMEOUT);
4338                 }
4339         }
4340
4341 link_up:
4342         spin_lock(&adapter->stats64_lock);
4343         e1000e_update_stats(adapter);
4344
4345         mac->tx_packet_delta = adapter->stats.tpt - adapter->tpt_old;
4346         adapter->tpt_old = adapter->stats.tpt;
4347         mac->collision_delta = adapter->stats.colc - adapter->colc_old;
4348         adapter->colc_old = adapter->stats.colc;
4349
4350         adapter->gorc = adapter->stats.gorc - adapter->gorc_old;
4351         adapter->gorc_old = adapter->stats.gorc;
4352         adapter->gotc = adapter->stats.gotc - adapter->gotc_old;
4353         adapter->gotc_old = adapter->stats.gotc;
4354         spin_unlock(&adapter->stats64_lock);
4355
4356         e1000e_update_adaptive(&adapter->hw);
4357
4358         if (!netif_carrier_ok(netdev) &&
4359             (e1000_desc_unused(tx_ring) + 1 < tx_ring->count)) {
4360                 /*
4361                  * We've lost link, so the controller stops DMA,
4362                  * but we've got queued Tx work that's never going
4363                  * to get done, so reset controller to flush Tx.
4364                  * (Do the reset outside of interrupt context).
4365                  */
4366                 schedule_work(&adapter->reset_task);
4367                 /* return immediately since reset is imminent */
4368                 return;
4369         }
4370
4371         /* Simple mode for Interrupt Throttle Rate (ITR) */
4372         if (adapter->itr_setting == 4) {
4373                 /*
4374                  * Symmetric Tx/Rx gets a reduced ITR=2000;
4375                  * Total asymmetrical Tx or Rx gets ITR=8000;
4376                  * everyone else is between 2000-8000.
4377                  */
4378                 u32 goc = (adapter->gotc + adapter->gorc) / 10000;
4379                 u32 dif = (adapter->gotc > adapter->gorc ?
4380                             adapter->gotc - adapter->gorc :
4381                             adapter->gorc - adapter->gotc) / 10000;
4382                 u32 itr = goc > 0 ? (dif * 6000 / goc + 2000) : 8000;
4383
4384                 ew32(ITR, 1000000000 / (itr * 256));
4385         }
4386
4387         /* Cause software interrupt to ensure Rx ring is cleaned */
4388         if (adapter->msix_entries)
4389                 ew32(ICS, adapter->rx_ring->ims_val);
4390         else
4391                 ew32(ICS, E1000_ICS_RXDMT0);
4392
4393         /* flush pending descriptors to memory before detecting Tx hang */
4394         e1000e_flush_descriptors(adapter);
4395
4396         /* Force detection of hung controller every watchdog period */
4397         adapter->detect_tx_hung = 1;
4398
4399         /*
4400          * With 82571 controllers, LAA may be overwritten due to controller
4401          * reset from the other port. Set the appropriate LAA in RAR[0]
4402          */
4403         if (e1000e_get_laa_state_82571(hw))
4404                 e1000e_rar_set(hw, adapter->hw.mac.addr, 0);
4405
4406         if (adapter->flags2 & FLAG2_CHECK_PHY_HANG)
4407                 e1000e_check_82574_phy_workaround(adapter);
4408
4409         /* Reset the timer */
4410         if (!test_bit(__E1000_DOWN, &adapter->state))
4411                 mod_timer(&adapter->watchdog_timer,
4412                           round_jiffies(jiffies + 2 * HZ));
4413 }
4414
4415 #define E1000_TX_FLAGS_CSUM             0x00000001
4416 #define E1000_TX_FLAGS_VLAN             0x00000002
4417 #define E1000_TX_FLAGS_TSO              0x00000004
4418 #define E1000_TX_FLAGS_IPV4             0x00000008
4419 #define E1000_TX_FLAGS_VLAN_MASK        0xffff0000
4420 #define E1000_TX_FLAGS_VLAN_SHIFT       16
4421
4422 static int e1000_tso(struct e1000_adapter *adapter,
4423                      struct sk_buff *skb)
4424 {
4425         struct e1000_ring *tx_ring = adapter->tx_ring;
4426         struct e1000_context_desc *context_desc;
4427         struct e1000_buffer *buffer_info;
4428         unsigned int i;
4429         u32 cmd_length = 0;
4430         u16 ipcse = 0, tucse, mss;
4431         u8 ipcss, ipcso, tucss, tucso, hdr_len;
4432
4433         if (!skb_is_gso(skb))
4434                 return 0;
4435
4436         if (skb_header_cloned(skb)) {
4437                 int err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
4438
4439                 if (err)
4440                         return err;
4441         }
4442
4443         hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
4444         mss = skb_shinfo(skb)->gso_size;
4445         if (skb->protocol == htons(ETH_P_IP)) {
4446                 struct iphdr *iph = ip_hdr(skb);
4447                 iph->tot_len = 0;
4448                 iph->check = 0;
4449                 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr,
4450                                                          0, IPPROTO_TCP, 0);
4451                 cmd_length = E1000_TXD_CMD_IP;
4452                 ipcse = skb_transport_offset(skb) - 1;
4453         } else if (skb_is_gso_v6(skb)) {
4454                 ipv6_hdr(skb)->payload_len = 0;
4455                 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
4456                                                        &ipv6_hdr(skb)->daddr,
4457                                                        0, IPPROTO_TCP, 0);
4458                 ipcse = 0;
4459         }
4460         ipcss = skb_network_offset(skb);
4461         ipcso = (void *)&(ip_hdr(skb)->check) - (void *)skb->data;
4462         tucss = skb_transport_offset(skb);
4463         tucso = (void *)&(tcp_hdr(skb)->check) - (void *)skb->data;
4464         tucse = 0;
4465
4466         cmd_length |= (E1000_TXD_CMD_DEXT | E1000_TXD_CMD_TSE |
4467                        E1000_TXD_CMD_TCP | (skb->len - (hdr_len)));
4468
4469         i = tx_ring->next_to_use;
4470         context_desc = E1000_CONTEXT_DESC(*tx_ring, i);
4471         buffer_info = &tx_ring->buffer_info[i];
4472
4473         context_desc->lower_setup.ip_fields.ipcss  = ipcss;
4474         context_desc->lower_setup.ip_fields.ipcso  = ipcso;
4475         context_desc->lower_setup.ip_fields.ipcse  = cpu_to_le16(ipcse);
4476         context_desc->upper_setup.tcp_fields.tucss = tucss;
4477         context_desc->upper_setup.tcp_fields.tucso = tucso;
4478         context_desc->upper_setup.tcp_fields.tucse = cpu_to_le16(tucse);
4479         context_desc->tcp_seg_setup.fields.mss     = cpu_to_le16(mss);
4480         context_desc->tcp_seg_setup.fields.hdr_len = hdr_len;
4481         context_desc->cmd_and_length = cpu_to_le32(cmd_length);
4482
4483         buffer_info->time_stamp = jiffies;
4484         buffer_info->next_to_watch = i;
4485
4486         i++;
4487         if (i == tx_ring->count)
4488                 i = 0;
4489         tx_ring->next_to_use = i;
4490
4491         return 1;
4492 }
4493
4494 static bool e1000_tx_csum(struct e1000_adapter *adapter, struct sk_buff *skb)
4495 {
4496         struct e1000_ring *tx_ring = adapter->tx_ring;
4497         struct e1000_context_desc *context_desc;
4498         struct e1000_buffer *buffer_info;
4499         unsigned int i;
4500         u8 css;
4501         u32 cmd_len = E1000_TXD_CMD_DEXT;
4502         __be16 protocol;
4503
4504         if (skb->ip_summed != CHECKSUM_PARTIAL)
4505                 return 0;
4506
4507         if (skb->protocol == cpu_to_be16(ETH_P_8021Q))
4508                 protocol = vlan_eth_hdr(skb)->h_vlan_encapsulated_proto;
4509         else
4510                 protocol = skb->protocol;
4511
4512         switch (protocol) {
4513         case cpu_to_be16(ETH_P_IP):
4514                 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
4515                         cmd_len |= E1000_TXD_CMD_TCP;
4516                 break;
4517         case cpu_to_be16(ETH_P_IPV6):
4518                 /* XXX not handling all IPV6 headers */
4519                 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
4520                         cmd_len |= E1000_TXD_CMD_TCP;
4521                 break;
4522         default:
4523                 if (unlikely(net_ratelimit()))
4524                         e_warn("checksum_partial proto=%x!\n",
4525                                be16_to_cpu(protocol));
4526                 break;
4527         }
4528
4529         css = skb_checksum_start_offset(skb);
4530
4531         i = tx_ring->next_to_use;
4532         buffer_info = &tx_ring->buffer_info[i];
4533         context_desc = E1000_CONTEXT_DESC(*tx_ring, i);
4534
4535         context_desc->lower_setup.ip_config = 0;
4536         context_desc->upper_setup.tcp_fields.tucss = css;
4537         context_desc->upper_setup.tcp_fields.tucso =
4538                                 css + skb->csum_offset;
4539         context_desc->upper_setup.tcp_fields.tucse = 0;
4540         context_desc->tcp_seg_setup.data = 0;
4541         context_desc->cmd_and_length = cpu_to_le32(cmd_len);
4542
4543         buffer_info->time_stamp = jiffies;
4544         buffer_info->next_to_watch = i;
4545
4546         i++;
4547         if (i == tx_ring->count)
4548                 i = 0;
4549         tx_ring->next_to_use = i;
4550
4551         return 1;
4552 }
4553
4554 #define E1000_MAX_PER_TXD       8192
4555 #define E1000_MAX_TXD_PWR       12
4556
4557 static int e1000_tx_map(struct e1000_adapter *adapter,
4558                         struct sk_buff *skb, unsigned int first,
4559                         unsigned int max_per_txd, unsigned int nr_frags,
4560                         unsigned int mss)
4561 {
4562         struct e1000_ring *tx_ring = adapter->tx_ring;
4563         struct pci_dev *pdev = adapter->pdev;
4564         struct e1000_buffer *buffer_info;
4565         unsigned int len = skb_headlen(skb);
4566         unsigned int offset = 0, size, count = 0, i;
4567         unsigned int f, bytecount, segs;
4568
4569         i = tx_ring->next_to_use;
4570
4571         while (len) {
4572                 buffer_info = &tx_ring->buffer_info[i];
4573                 size = min(len, max_per_txd);
4574
4575                 buffer_info->length = size;
4576                 buffer_info->time_stamp = jiffies;
4577                 buffer_info->next_to_watch = i;
4578                 buffer_info->dma = dma_map_single(&pdev->dev,
4579                                                   skb->data + offset,
4580                                                   size, DMA_TO_DEVICE);
4581                 buffer_info->mapped_as_page = false;
4582                 if (dma_mapping_error(&pdev->dev, buffer_info->dma))
4583                         goto dma_error;
4584
4585                 len -= size;
4586                 offset += size;
4587                 count++;
4588
4589                 if (len) {
4590                         i++;
4591                         if (i == tx_ring->count)
4592                                 i = 0;
4593                 }
4594         }
4595
4596         for (f = 0; f < nr_frags; f++) {
4597                 struct skb_frag_struct *frag;
4598
4599                 frag = &skb_shinfo(skb)->frags[f];
4600                 len = frag->size;
4601                 offset = frag->page_offset;
4602
4603                 while (len) {
4604                         i++;
4605                         if (i == tx_ring->count)
4606                                 i = 0;
4607
4608                         buffer_info = &tx_ring->buffer_info[i];
4609                         size = min(len, max_per_txd);
4610
4611                         buffer_info->length = size;
4612                         buffer_info->time_stamp = jiffies;
4613                         buffer_info->next_to_watch = i;
4614                         buffer_info->dma = dma_map_page(&pdev->dev, frag->page,
4615                                                         offset, size,
4616                                                         DMA_TO_DEVICE);
4617                         buffer_info->mapped_as_page = true;
4618                         if (dma_mapping_error(&pdev->dev, buffer_info->dma))
4619                                 goto dma_error;
4620
4621                         len -= size;
4622                         offset += size;
4623                         count++;
4624                 }
4625         }
4626
4627         segs = skb_shinfo(skb)->gso_segs ? : 1;
4628         /* multiply data chunks by size of headers */
4629         bytecount = ((segs - 1) * skb_headlen(skb)) + skb->len;
4630
4631         tx_ring->buffer_info[i].skb = skb;
4632         tx_ring->buffer_info[i].segs = segs;
4633         tx_ring->buffer_info[i].bytecount = bytecount;
4634         tx_ring->buffer_info[first].next_to_watch = i;
4635
4636         return count;
4637
4638 dma_error:
4639         dev_err(&pdev->dev, "Tx DMA map failed\n");
4640         buffer_info->dma = 0;
4641         if (count)
4642                 count--;
4643
4644         while (count--) {
4645                 if (i == 0)
4646                         i += tx_ring->count;
4647                 i--;
4648                 buffer_info = &tx_ring->buffer_info[i];
4649                 e1000_put_txbuf(adapter, buffer_info);
4650         }
4651
4652         return 0;
4653 }
4654
4655 static void e1000_tx_queue(struct e1000_adapter *adapter,
4656                            int tx_flags, int count)
4657 {
4658         struct e1000_ring *tx_ring = adapter->tx_ring;
4659         struct e1000_tx_desc *tx_desc = NULL;
4660         struct e1000_buffer *buffer_info;
4661         u32 txd_upper = 0, txd_lower = E1000_TXD_CMD_IFCS;
4662         unsigned int i;
4663
4664         if (tx_flags & E1000_TX_FLAGS_TSO) {
4665                 txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D |
4666                              E1000_TXD_CMD_TSE;
4667                 txd_upper |= E1000_TXD_POPTS_TXSM << 8;
4668
4669                 if (tx_flags & E1000_TX_FLAGS_IPV4)
4670                         txd_upper |= E1000_TXD_POPTS_IXSM << 8;
4671         }
4672
4673         if (tx_flags & E1000_TX_FLAGS_CSUM) {
4674                 txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
4675                 txd_upper |= E1000_TXD_POPTS_TXSM << 8;
4676         }
4677
4678         if (tx_flags & E1000_TX_FLAGS_VLAN) {
4679                 txd_lower |= E1000_TXD_CMD_VLE;
4680                 txd_upper |= (tx_flags & E1000_TX_FLAGS_VLAN_MASK);
4681         }
4682
4683         i = tx_ring->next_to_use;
4684
4685         do {
4686                 buffer_info = &tx_ring->buffer_info[i];
4687                 tx_desc = E1000_TX_DESC(*tx_ring, i);
4688                 tx_desc->buffer_addr = cpu_to_le64(buffer_info->dma);
4689                 tx_desc->lower.data =
4690                         cpu_to_le32(txd_lower | buffer_info->length);
4691                 tx_desc->upper.data = cpu_to_le32(txd_upper);
4692
4693                 i++;
4694                 if (i == tx_ring->count)
4695                         i = 0;
4696         } while (--count > 0);
4697
4698         tx_desc->lower.data |= cpu_to_le32(adapter->txd_cmd);
4699
4700         /*
4701          * Force memory writes to complete before letting h/w
4702          * know there are new descriptors to fetch.  (Only
4703          * applicable for weak-ordered memory model archs,
4704          * such as IA-64).
4705          */
4706         wmb();
4707
4708         tx_ring->next_to_use = i;
4709         writel(i, adapter->hw.hw_addr + tx_ring->tail);
4710         /*
4711          * we need this if more than one processor can write to our tail
4712          * at a time, it synchronizes IO on IA64/Altix systems
4713          */
4714         mmiowb();
4715 }
4716
4717 #define MINIMUM_DHCP_PACKET_SIZE 282
4718 static int e1000_transfer_dhcp_info(struct e1000_adapter *adapter,
4719                                     struct sk_buff *skb)
4720 {
4721         struct e1000_hw *hw =  &adapter->hw;
4722         u16 length, offset;
4723
4724         if (vlan_tx_tag_present(skb)) {
4725                 if (!((vlan_tx_tag_get(skb) == adapter->hw.mng_cookie.vlan_id) &&
4726                     (adapter->hw.mng_cookie.status &
4727                         E1000_MNG_DHCP_COOKIE_STATUS_VLAN)))
4728                         return 0;
4729         }
4730
4731         if (skb->len <= MINIMUM_DHCP_PACKET_SIZE)
4732                 return 0;
4733
4734         if (((struct ethhdr *) skb->data)->h_proto != htons(ETH_P_IP))
4735                 return 0;
4736
4737         {
4738                 const struct iphdr *ip = (struct iphdr *)((u8 *)skb->data+14);
4739                 struct udphdr *udp;
4740
4741                 if (ip->protocol != IPPROTO_UDP)
4742                         return 0;
4743
4744                 udp = (struct udphdr *)((u8 *)ip + (ip->ihl << 2));
4745                 if (ntohs(udp->dest) != 67)
4746                         return 0;
4747
4748                 offset = (u8 *)udp + 8 - skb->data;
4749                 length = skb->len - offset;
4750                 return e1000e_mng_write_dhcp_info(hw, (u8 *)udp + 8, length);
4751         }
4752
4753         return 0;
4754 }
4755
4756 static int __e1000_maybe_stop_tx(struct net_device *netdev, int size)
4757 {
4758         struct e1000_adapter *adapter = netdev_priv(netdev);
4759
4760         netif_stop_queue(netdev);
4761         /*
4762          * Herbert's original patch had:
4763          *  smp_mb__after_netif_stop_queue();
4764          * but since that doesn't exist yet, just open code it.
4765          */
4766         smp_mb();
4767
4768         /*
4769          * We need to check again in a case another CPU has just
4770          * made room available.
4771          */
4772         if (e1000_desc_unused(adapter->tx_ring) < size)
4773                 return -EBUSY;
4774
4775         /* A reprieve! */
4776         netif_start_queue(netdev);
4777         ++adapter->restart_queue;
4778         return 0;
4779 }
4780
4781 static int e1000_maybe_stop_tx(struct net_device *netdev, int size)
4782 {
4783         struct e1000_adapter *adapter = netdev_priv(netdev);
4784
4785         if (e1000_desc_unused(adapter->tx_ring) >= size)
4786                 return 0;
4787         return __e1000_maybe_stop_tx(netdev, size);
4788 }
4789
4790 #define TXD_USE_COUNT(S, X) (((S) >> (X)) + 1 )
4791 static netdev_tx_t e1000_xmit_frame(struct sk_buff *skb,
4792                                     struct net_device *netdev)
4793 {
4794         struct e1000_adapter *adapter = netdev_priv(netdev);
4795         struct e1000_ring *tx_ring = adapter->tx_ring;
4796         unsigned int first;
4797         unsigned int max_per_txd = E1000_MAX_PER_TXD;
4798         unsigned int max_txd_pwr = E1000_MAX_TXD_PWR;
4799         unsigned int tx_flags = 0;
4800         unsigned int len = skb_headlen(skb);
4801         unsigned int nr_frags;
4802         unsigned int mss;
4803         int count = 0;
4804         int tso;
4805         unsigned int f;
4806
4807         if (test_bit(__E1000_DOWN, &adapter->state)) {
4808                 dev_kfree_skb_any(skb);
4809                 return NETDEV_TX_OK;
4810         }
4811
4812         if (skb->len <= 0) {
4813                 dev_kfree_skb_any(skb);
4814                 return NETDEV_TX_OK;
4815         }
4816
4817         mss = skb_shinfo(skb)->gso_size;
4818         /*
4819          * The controller does a simple calculation to
4820          * make sure there is enough room in the FIFO before
4821          * initiating the DMA for each buffer.  The calc is:
4822          * 4 = ceil(buffer len/mss).  To make sure we don't
4823          * overrun the FIFO, adjust the max buffer len if mss
4824          * drops.
4825          */
4826         if (mss) {
4827                 u8 hdr_len;
4828                 max_per_txd = min(mss << 2, max_per_txd);
4829                 max_txd_pwr = fls(max_per_txd) - 1;
4830
4831                 /*
4832                  * TSO Workaround for 82571/2/3 Controllers -- if skb->data
4833                  * points to just header, pull a few bytes of payload from
4834                  * frags into skb->data
4835                  */
4836                 hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
4837                 /*
4838                  * we do this workaround for ES2LAN, but it is un-necessary,
4839                  * avoiding it could save a lot of cycles
4840                  */
4841                 if (skb->data_len && (hdr_len == len)) {
4842                         unsigned int pull_size;
4843
4844                         pull_size = min((unsigned int)4, skb->data_len);
4845                         if (!__pskb_pull_tail(skb, pull_size)) {
4846                                 e_err("__pskb_pull_tail failed.\n");
4847                                 dev_kfree_skb_any(skb);
4848                                 return NETDEV_TX_OK;
4849                         }
4850                         len = skb_headlen(skb);
4851                 }
4852         }
4853
4854         /* reserve a descriptor for the offload context */
4855         if ((mss) || (skb->ip_summed == CHECKSUM_PARTIAL))
4856                 count++;
4857         count++;
4858
4859         count += TXD_USE_COUNT(len, max_txd_pwr);
4860
4861         nr_frags = skb_shinfo(skb)->nr_frags;
4862         for (f = 0; f < nr_frags; f++)
4863                 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size,
4864                                        max_txd_pwr);
4865
4866         if (adapter->hw.mac.tx_pkt_filtering)
4867                 e1000_transfer_dhcp_info(adapter, skb);
4868
4869         /*
4870          * need: count + 2 desc gap to keep tail from touching
4871          * head, otherwise try next time
4872          */
4873         if (e1000_maybe_stop_tx(netdev, count + 2))
4874                 return NETDEV_TX_BUSY;
4875
4876         if (vlan_tx_tag_present(skb)) {
4877                 tx_flags |= E1000_TX_FLAGS_VLAN;
4878                 tx_flags |= (vlan_tx_tag_get(skb) << E1000_TX_FLAGS_VLAN_SHIFT);
4879         }
4880
4881         first = tx_ring->next_to_use;
4882
4883         tso = e1000_tso(adapter, skb);
4884         if (tso < 0) {
4885                 dev_kfree_skb_any(skb);
4886                 return NETDEV_TX_OK;
4887         }
4888
4889         if (tso)
4890                 tx_flags |= E1000_TX_FLAGS_TSO;
4891         else if (e1000_tx_csum(adapter, skb))
4892                 tx_flags |= E1000_TX_FLAGS_CSUM;
4893
4894         /*
4895          * Old method was to assume IPv4 packet by default if TSO was enabled.
4896          * 82571 hardware supports TSO capabilities for IPv6 as well...
4897          * no longer assume, we must.
4898          */
4899         if (skb->protocol == htons(ETH_P_IP))
4900                 tx_flags |= E1000_TX_FLAGS_IPV4;
4901
4902         /* if count is 0 then mapping error has occurred */
4903         count = e1000_tx_map(adapter, skb, first, max_per_txd, nr_frags, mss);
4904         if (count) {
4905                 e1000_tx_queue(adapter, tx_flags, count);
4906                 /* Make sure there is space in the ring for the next send. */
4907                 e1000_maybe_stop_tx(netdev, MAX_SKB_FRAGS + 2);
4908
4909         } else {
4910                 dev_kfree_skb_any(skb);
4911                 tx_ring->buffer_info[first].time_stamp = 0;
4912                 tx_ring->next_to_use = first;
4913         }
4914
4915         return NETDEV_TX_OK;
4916 }
4917
4918 /**
4919  * e1000_tx_timeout - Respond to a Tx Hang
4920  * @netdev: network interface device structure
4921  **/
4922 static void e1000_tx_timeout(struct net_device *netdev)
4923 {
4924         struct e1000_adapter *adapter = netdev_priv(netdev);
4925
4926         /* Do the reset outside of interrupt context */
4927         adapter->tx_timeout_count++;
4928         schedule_work(&adapter->reset_task);
4929 }
4930
4931 static void e1000_reset_task(struct work_struct *work)
4932 {
4933         struct e1000_adapter *adapter;
4934         adapter = container_of(work, struct e1000_adapter, reset_task);
4935
4936         /* don't run the task if already down */
4937         if (test_bit(__E1000_DOWN, &adapter->state))
4938                 return;
4939
4940         if (!((adapter->flags & FLAG_RX_NEEDS_RESTART) &&
4941               (adapter->flags & FLAG_RX_RESTART_NOW))) {
4942                 e1000e_dump(adapter);
4943                 e_err("Reset adapter\n");
4944         }
4945         e1000e_reinit_locked(adapter);
4946 }
4947
4948 /**
4949  * e1000_get_stats64 - Get System Network Statistics
4950  * @netdev: network interface device structure
4951  * @stats: rtnl_link_stats64 pointer
4952  *
4953  * Returns the address of the device statistics structure.
4954  **/
4955 struct rtnl_link_stats64 *e1000e_get_stats64(struct net_device *netdev,
4956                                              struct rtnl_link_stats64 *stats)
4957 {
4958         struct e1000_adapter *adapter = netdev_priv(netdev);
4959
4960         memset(stats, 0, sizeof(struct rtnl_link_stats64));
4961         spin_lock(&adapter->stats64_lock);
4962         e1000e_update_stats(adapter);
4963         /* Fill out the OS statistics structure */
4964         stats->rx_bytes = adapter->stats.gorc;
4965         stats->rx_packets = adapter->stats.gprc;
4966         stats->tx_bytes = adapter->stats.gotc;
4967         stats->tx_packets = adapter->stats.gptc;
4968         stats->multicast = adapter->stats.mprc;
4969         stats->collisions = adapter->stats.colc;
4970
4971         /* Rx Errors */
4972
4973         /*
4974          * RLEC on some newer hardware can be incorrect so build
4975          * our own version based on RUC and ROC
4976          */
4977         stats->rx_errors = adapter->stats.rxerrc +
4978                 adapter->stats.crcerrs + adapter->stats.algnerrc +
4979                 adapter->stats.ruc + adapter->stats.roc +
4980                 adapter->stats.cexterr;
4981         stats->rx_length_errors = adapter->stats.ruc +
4982                                               adapter->stats.roc;
4983         stats->rx_crc_errors = adapter->stats.crcerrs;
4984         stats->rx_frame_errors = adapter->stats.algnerrc;
4985         stats->rx_missed_errors = adapter->stats.mpc;
4986
4987         /* Tx Errors */
4988         stats->tx_errors = adapter->stats.ecol +
4989                                        adapter->stats.latecol;
4990         stats->tx_aborted_errors = adapter->stats.ecol;
4991         stats->tx_window_errors = adapter->stats.latecol;
4992         stats->tx_carrier_errors = adapter->stats.tncrs;
4993
4994         /* Tx Dropped needs to be maintained elsewhere */
4995
4996         spin_unlock(&adapter->stats64_lock);
4997         return stats;
4998 }
4999
5000 /**
5001  * e1000_change_mtu - Change the Maximum Transfer Unit
5002  * @netdev: network interface device structure
5003  * @new_mtu: new value for maximum frame size
5004  *
5005  * Returns 0 on success, negative on failure
5006  **/
5007 static int e1000_change_mtu(struct net_device *netdev, int new_mtu)
5008 {
5009         struct e1000_adapter *adapter = netdev_priv(netdev);
5010         int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
5011
5012         /* Jumbo frame support */
5013         if ((max_frame > ETH_FRAME_LEN + ETH_FCS_LEN) &&
5014             !(adapter->flags & FLAG_HAS_JUMBO_FRAMES)) {
5015                 e_err("Jumbo Frames not supported.\n");
5016                 return -EINVAL;
5017         }
5018
5019         /* Supported frame sizes */
5020         if ((new_mtu < ETH_ZLEN + ETH_FCS_LEN + VLAN_HLEN) ||
5021             (max_frame > adapter->max_hw_frame_size)) {
5022                 e_err("Unsupported MTU setting\n");
5023                 return -EINVAL;
5024         }
5025
5026         /* Jumbo frame workaround on 82579 requires CRC be stripped */
5027         if ((adapter->hw.mac.type == e1000_pch2lan) &&
5028             !(adapter->flags2 & FLAG2_CRC_STRIPPING) &&
5029             (new_mtu > ETH_DATA_LEN)) {
5030                 e_err("Jumbo Frames not supported on 82579 when CRC "
5031                       "stripping is disabled.\n");
5032                 return -EINVAL;
5033         }
5034
5035         /* 82573 Errata 17 */
5036         if (((adapter->hw.mac.type == e1000_82573) ||
5037              (adapter->hw.mac.type == e1000_82574)) &&
5038             (max_frame > ETH_FRAME_LEN + ETH_FCS_LEN)) {
5039                 adapter->flags2 |= FLAG2_DISABLE_ASPM_L1;
5040                 e1000e_disable_aspm(adapter->pdev, PCIE_LINK_STATE_L1);
5041         }
5042
5043         while (test_and_set_bit(__E1000_RESETTING, &adapter->state))
5044                 usleep_range(1000, 2000);
5045         /* e1000e_down -> e1000e_reset dependent on max_frame_size & mtu */
5046         adapter->max_frame_size = max_frame;
5047         e_info("changing MTU from %d to %d\n", netdev->mtu, new_mtu);
5048         netdev->mtu = new_mtu;
5049         if (netif_running(netdev))
5050                 e1000e_down(adapter);
5051
5052         /*
5053          * NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
5054          * means we reserve 2 more, this pushes us to allocate from the next
5055          * larger slab size.
5056          * i.e. RXBUFFER_2048 --> size-4096 slab
5057          * However with the new *_jumbo_rx* routines, jumbo receives will use
5058          * fragmented skbs
5059          */
5060
5061         if (max_frame <= 2048)
5062                 adapter->rx_buffer_len = 2048;
5063         else
5064                 adapter->rx_buffer_len = 4096;
5065
5066         /* adjust allocation if LPE protects us, and we aren't using SBP */
5067         if ((max_frame == ETH_FRAME_LEN + ETH_FCS_LEN) ||
5068              (max_frame == ETH_FRAME_LEN + VLAN_HLEN + ETH_FCS_LEN))
5069                 adapter->rx_buffer_len = ETH_FRAME_LEN + VLAN_HLEN
5070                                          + ETH_FCS_LEN;
5071
5072         if (netif_running(netdev))
5073                 e1000e_up(adapter);
5074         else
5075                 e1000e_reset(adapter);
5076
5077         clear_bit(__E1000_RESETTING, &adapter->state);
5078
5079         return 0;
5080 }
5081
5082 static int e1000_mii_ioctl(struct net_device *netdev, struct ifreq *ifr,
5083                            int cmd)
5084 {
5085         struct e1000_adapter *adapter = netdev_priv(netdev);
5086         struct mii_ioctl_data *data = if_mii(ifr);
5087
5088         if (adapter->hw.phy.media_type != e1000_media_type_copper)
5089                 return -EOPNOTSUPP;
5090
5091         switch (cmd) {
5092         case SIOCGMIIPHY:
5093                 data->phy_id = adapter->hw.phy.addr;
5094                 break;
5095         case SIOCGMIIREG:
5096                 e1000_phy_read_status(adapter);
5097
5098                 switch (data->reg_num & 0x1F) {
5099                 case MII_BMCR:
5100                         data->val_out = adapter->phy_regs.bmcr;
5101                         break;
5102                 case MII_BMSR:
5103                         data->val_out = adapter->phy_regs.bmsr;
5104                         break;
5105                 case MII_PHYSID1:
5106                         data->val_out = (adapter->hw.phy.id >> 16);
5107                         break;
5108                 case MII_PHYSID2:
5109                         data->val_out = (adapter->hw.phy.id & 0xFFFF);
5110                         break;
5111                 case MII_ADVERTISE:
5112                         data->val_out = adapter->phy_regs.advertise;
5113                         break;
5114                 case MII_LPA:
5115                         data->val_out = adapter->phy_regs.lpa;
5116                         break;
5117                 case MII_EXPANSION:
5118                         data->val_out = adapter->phy_regs.expansion;
5119                         break;
5120                 case MII_CTRL1000:
5121                         data->val_out = adapter->phy_regs.ctrl1000;
5122                         break;
5123                 case MII_STAT1000:
5124                         data->val_out = adapter->phy_regs.stat1000;
5125                         break;
5126                 case MII_ESTATUS:
5127                         data->val_out = adapter->phy_regs.estatus;
5128                         break;
5129                 default:
5130                         return -EIO;
5131                 }
5132                 break;
5133         case SIOCSMIIREG:
5134         default:
5135                 return -EOPNOTSUPP;
5136         }
5137         return 0;
5138 }
5139
5140 static int e1000_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5141 {
5142         switch (cmd) {
5143         case SIOCGMIIPHY:
5144         case SIOCGMIIREG:
5145         case SIOCSMIIREG:
5146                 return e1000_mii_ioctl(netdev, ifr, cmd);
5147         default:
5148                 return -EOPNOTSUPP;
5149         }
5150 }
5151
5152 static int e1000_init_phy_wakeup(struct e1000_adapter *adapter, u32 wufc)
5153 {
5154         struct e1000_hw *hw = &adapter->hw;
5155         u32 i, mac_reg;
5156         u16 phy_reg;
5157         int retval = 0;
5158
5159         /* copy MAC RARs to PHY RARs */
5160         e1000_copy_rx_addrs_to_phy_ich8lan(hw);
5161
5162         /* copy MAC MTA to PHY MTA */
5163         for (i = 0; i < adapter->hw.mac.mta_reg_count; i++) {
5164                 mac_reg = E1000_READ_REG_ARRAY(hw, E1000_MTA, i);
5165                 e1e_wphy(hw, BM_MTA(i), (u16)(mac_reg & 0xFFFF));
5166                 e1e_wphy(hw, BM_MTA(i) + 1, (u16)((mac_reg >> 16) & 0xFFFF));
5167         }
5168
5169         /* configure PHY Rx Control register */
5170         e1e_rphy(&adapter->hw, BM_RCTL, &phy_reg);
5171         mac_reg = er32(RCTL);
5172         if (mac_reg & E1000_RCTL_UPE)
5173                 phy_reg |= BM_RCTL_UPE;
5174         if (mac_reg & E1000_RCTL_MPE)
5175                 phy_reg |= BM_RCTL_MPE;
5176         phy_reg &= ~(BM_RCTL_MO_MASK);
5177         if (mac_reg & E1000_RCTL_MO_3)
5178                 phy_reg |= (((mac_reg & E1000_RCTL_MO_3) >> E1000_RCTL_MO_SHIFT)
5179                                 << BM_RCTL_MO_SHIFT);
5180         if (mac_reg & E1000_RCTL_BAM)
5181                 phy_reg |= BM_RCTL_BAM;
5182         if (mac_reg & E1000_RCTL_PMCF)
5183                 phy_reg |= BM_RCTL_PMCF;
5184         mac_reg = er32(CTRL);
5185         if (mac_reg & E1000_CTRL_RFCE)
5186                 phy_reg |= BM_RCTL_RFCE;
5187         e1e_wphy(&adapter->hw, BM_RCTL, phy_reg);
5188
5189         /* enable PHY wakeup in MAC register */
5190         ew32(WUFC, wufc);
5191         ew32(WUC, E1000_WUC_PHY_WAKE | E1000_WUC_PME_EN);
5192
5193         /* configure and enable PHY wakeup in PHY registers */
5194         e1e_wphy(&adapter->hw, BM_WUFC, wufc);
5195         e1e_wphy(&adapter->hw, BM_WUC, E1000_WUC_PME_EN);
5196
5197         /* activate PHY wakeup */
5198         retval = hw->phy.ops.acquire(hw);
5199         if (retval) {
5200                 e_err("Could not acquire PHY\n");
5201                 return retval;
5202         }
5203         e1000e_write_phy_reg_mdic(hw, IGP01E1000_PHY_PAGE_SELECT,
5204                                  (BM_WUC_ENABLE_PAGE << IGP_PAGE_SHIFT));
5205         retval = e1000e_read_phy_reg_mdic(hw, BM_WUC_ENABLE_REG, &phy_reg);
5206         if (retval) {
5207                 e_err("Could not read PHY page 769\n");
5208                 goto out;
5209         }
5210         phy_reg |= BM_WUC_ENABLE_BIT | BM_WUC_HOST_WU_BIT;
5211         retval = e1000e_write_phy_reg_mdic(hw, BM_WUC_ENABLE_REG, phy_reg);
5212         if (retval)
5213                 e_err("Could not set PHY Host Wakeup bit\n");
5214 out:
5215         hw->phy.ops.release(hw);
5216
5217         return retval;
5218 }
5219
5220 static int __e1000_shutdown(struct pci_dev *pdev, bool *enable_wake,
5221                             bool runtime)
5222 {
5223         struct net_device *netdev = pci_get_drvdata(pdev);
5224         struct e1000_adapter *adapter = netdev_priv(netdev);
5225         struct e1000_hw *hw = &adapter->hw;
5226         u32 ctrl, ctrl_ext, rctl, status;
5227         /* Runtime suspend should only enable wakeup for link changes */
5228         u32 wufc = runtime ? E1000_WUFC_LNKC : adapter->wol;
5229         int retval = 0;
5230
5231         netif_device_detach(netdev);
5232
5233         if (netif_running(netdev)) {
5234                 WARN_ON(test_bit(__E1000_RESETTING, &adapter->state));
5235                 e1000e_down(adapter);
5236                 e1000_free_irq(adapter);
5237         }
5238         e1000e_reset_interrupt_capability(adapter);
5239
5240         retval = pci_save_state(pdev);
5241         if (retval)
5242                 return retval;
5243
5244         status = er32(STATUS);
5245         if (status & E1000_STATUS_LU)
5246                 wufc &= ~E1000_WUFC_LNKC;
5247
5248         if (wufc) {
5249                 e1000_setup_rctl(adapter);
5250                 e1000_set_multi(netdev);
5251
5252                 /* turn on all-multi mode if wake on multicast is enabled */
5253                 if (wufc & E1000_WUFC_MC) {
5254                         rctl = er32(RCTL);
5255                         rctl |= E1000_RCTL_MPE;
5256                         ew32(RCTL, rctl);
5257                 }
5258
5259                 ctrl = er32(CTRL);
5260                 /* advertise wake from D3Cold */
5261                 #define E1000_CTRL_ADVD3WUC 0x00100000
5262                 /* phy power management enable */
5263                 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
5264                 ctrl |= E1000_CTRL_ADVD3WUC;
5265                 if (!(adapter->flags2 & FLAG2_HAS_PHY_WAKEUP))
5266                         ctrl |= E1000_CTRL_EN_PHY_PWR_MGMT;
5267                 ew32(CTRL, ctrl);
5268
5269                 if (adapter->hw.phy.media_type == e1000_media_type_fiber ||
5270                     adapter->hw.phy.media_type ==
5271                     e1000_media_type_internal_serdes) {
5272                         /* keep the laser running in D3 */
5273                         ctrl_ext = er32(CTRL_EXT);
5274                         ctrl_ext |= E1000_CTRL_EXT_SDP3_DATA;
5275                         ew32(CTRL_EXT, ctrl_ext);
5276                 }
5277
5278                 if (adapter->flags & FLAG_IS_ICH)
5279                         e1000e_disable_gig_wol_ich8lan(&adapter->hw);
5280
5281                 /* Allow time for pending master requests to run */
5282                 e1000e_disable_pcie_master(&adapter->hw);
5283
5284                 if (adapter->flags2 & FLAG2_HAS_PHY_WAKEUP) {
5285                         /* enable wakeup by the PHY */
5286                         retval = e1000_init_phy_wakeup(adapter, wufc);
5287                         if (retval)
5288                                 return retval;
5289                 } else {
5290                         /* enable wakeup by the MAC */
5291                         ew32(WUFC, wufc);
5292                         ew32(WUC, E1000_WUC_PME_EN);
5293                 }
5294         } else {
5295                 ew32(WUC, 0);
5296                 ew32(WUFC, 0);
5297         }
5298
5299         *enable_wake = !!wufc;
5300
5301         /* make sure adapter isn't asleep if manageability is enabled */
5302         if ((adapter->flags & FLAG_MNG_PT_ENABLED) ||
5303             (hw->mac.ops.check_mng_mode(hw)))
5304                 *enable_wake = true;
5305
5306         if (adapter->hw.phy.type == e1000_phy_igp_3)
5307                 e1000e_igp3_phy_powerdown_workaround_ich8lan(&adapter->hw);
5308
5309         /*
5310          * Release control of h/w to f/w.  If f/w is AMT enabled, this
5311          * would have already happened in close and is redundant.
5312          */
5313         e1000e_release_hw_control(adapter);
5314
5315         pci_disable_device(pdev);
5316
5317         return 0;
5318 }
5319
5320 static void e1000_power_off(struct pci_dev *pdev, bool sleep, bool wake)
5321 {
5322         if (sleep && wake) {
5323                 pci_prepare_to_sleep(pdev);
5324                 return;
5325         }
5326
5327         pci_wake_from_d3(pdev, wake);
5328         pci_set_power_state(pdev, PCI_D3hot);
5329 }
5330
5331 static void e1000_complete_shutdown(struct pci_dev *pdev, bool sleep,
5332                                     bool wake)
5333 {
5334         struct net_device *netdev = pci_get_drvdata(pdev);
5335         struct e1000_adapter *adapter = netdev_priv(netdev);
5336
5337         /*
5338          * The pci-e switch on some quad port adapters will report a
5339          * correctable error when the MAC transitions from D0 to D3.  To
5340          * prevent this we need to mask off the correctable errors on the
5341          * downstream port of the pci-e switch.
5342          */
5343         if (adapter->flags & FLAG_IS_QUAD_PORT) {
5344                 struct pci_dev *us_dev = pdev->bus->self;
5345                 int pos = pci_find_capability(us_dev, PCI_CAP_ID_EXP);
5346                 u16 devctl;
5347
5348                 pci_read_config_word(us_dev, pos + PCI_EXP_DEVCTL, &devctl);
5349                 pci_write_config_word(us_dev, pos + PCI_EXP_DEVCTL,
5350                                       (devctl & ~PCI_EXP_DEVCTL_CERE));
5351
5352                 e1000_power_off(pdev, sleep, wake);
5353
5354                 pci_write_config_word(us_dev, pos + PCI_EXP_DEVCTL, devctl);
5355         } else {
5356                 e1000_power_off(pdev, sleep, wake);
5357         }
5358 }
5359
5360 #ifdef CONFIG_PCIEASPM
5361 static void __e1000e_disable_aspm(struct pci_dev *pdev, u16 state)
5362 {
5363         pci_disable_link_state(pdev, state);
5364 }
5365 #else
5366 static void __e1000e_disable_aspm(struct pci_dev *pdev, u16 state)
5367 {
5368         int pos;
5369         u16 reg16;
5370
5371         /*
5372          * Both device and parent should have the same ASPM setting.
5373          * Disable ASPM in downstream component first and then upstream.
5374          */
5375         pos = pci_pcie_cap(pdev);
5376         pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, &reg16);
5377         reg16 &= ~state;
5378         pci_write_config_word(pdev, pos + PCI_EXP_LNKCTL, reg16);
5379
5380         if (!pdev->bus->self)
5381                 return;
5382
5383         pos = pci_pcie_cap(pdev->bus->self);
5384         pci_read_config_word(pdev->bus->self, pos + PCI_EXP_LNKCTL, &reg16);
5385         reg16 &= ~state;
5386         pci_write_config_word(pdev->bus->self, pos + PCI_EXP_LNKCTL, reg16);
5387 }
5388 #endif
5389 static void e1000e_disable_aspm(struct pci_dev *pdev, u16 state)
5390 {
5391         dev_info(&pdev->dev, "Disabling ASPM %s %s\n",
5392                  (state & PCIE_LINK_STATE_L0S) ? "L0s" : "",
5393                  (state & PCIE_LINK_STATE_L1) ? "L1" : "");
5394
5395         __e1000e_disable_aspm(pdev, state);
5396 }
5397
5398 #ifdef CONFIG_PM
5399 static bool e1000e_pm_ready(struct e1000_adapter *adapter)
5400 {
5401         return !!adapter->tx_ring->buffer_info;
5402 }
5403
5404 static int __e1000_resume(struct pci_dev *pdev)
5405 {
5406         struct net_device *netdev = pci_get_drvdata(pdev);
5407         struct e1000_adapter *adapter = netdev_priv(netdev);
5408         struct e1000_hw *hw = &adapter->hw;
5409         u16 aspm_disable_flag = 0;
5410         u32 err;
5411
5412         if (adapter->flags2 & FLAG2_DISABLE_ASPM_L0S)
5413                 aspm_disable_flag = PCIE_LINK_STATE_L0S;
5414         if (adapter->flags2 & FLAG2_DISABLE_ASPM_L1)
5415                 aspm_disable_flag |= PCIE_LINK_STATE_L1;
5416         if (aspm_disable_flag)
5417                 e1000e_disable_aspm(pdev, aspm_disable_flag);
5418
5419         pci_set_power_state(pdev, PCI_D0);
5420         pci_restore_state(pdev);
5421         pci_save_state(pdev);
5422
5423         e1000e_set_interrupt_capability(adapter);
5424         if (netif_running(netdev)) {
5425                 err = e1000_request_irq(adapter);
5426                 if (err)
5427                         return err;
5428         }
5429
5430         e1000e_power_up_phy(adapter);
5431
5432         /* report the system wakeup cause from S3/S4 */
5433         if (adapter->flags2 & FLAG2_HAS_PHY_WAKEUP) {
5434                 u16 phy_data;
5435
5436                 e1e_rphy(&adapter->hw, BM_WUS, &phy_data);
5437                 if (phy_data) {
5438                         e_info("PHY Wakeup cause - %s\n",
5439                                 phy_data & E1000_WUS_EX ? "Unicast Packet" :
5440                                 phy_data & E1000_WUS_MC ? "Multicast Packet" :
5441                                 phy_data & E1000_WUS_BC ? "Broadcast Packet" :
5442                                 phy_data & E1000_WUS_MAG ? "Magic Packet" :
5443                                 phy_data & E1000_WUS_LNKC ? "Link Status "
5444                                 " Change" : "other");
5445                 }
5446                 e1e_wphy(&adapter->hw, BM_WUS, ~0);
5447         } else {
5448                 u32 wus = er32(WUS);
5449                 if (wus) {
5450                         e_info("MAC Wakeup cause - %s\n",
5451                                 wus & E1000_WUS_EX ? "Unicast Packet" :
5452                                 wus & E1000_WUS_MC ? "Multicast Packet" :
5453                                 wus & E1000_WUS_BC ? "Broadcast Packet" :
5454                                 wus & E1000_WUS_MAG ? "Magic Packet" :
5455                                 wus & E1000_WUS_LNKC ? "Link Status Change" :
5456                                 "other");
5457                 }
5458                 ew32(WUS, ~0);
5459         }
5460
5461         e1000e_reset(adapter);
5462
5463         e1000_init_manageability_pt(adapter);
5464
5465         if (netif_running(netdev))
5466                 e1000e_up(adapter);
5467
5468         netif_device_attach(netdev);
5469
5470         /*
5471          * If the controller has AMT, do not set DRV_LOAD until the interface
5472          * is up.  For all other cases, let the f/w know that the h/w is now
5473          * under the control of the driver.
5474          */
5475         if (!(adapter->flags & FLAG_HAS_AMT))
5476                 e1000e_get_hw_control(adapter);
5477
5478         return 0;
5479 }
5480
5481 #ifdef CONFIG_PM_SLEEP
5482 static int e1000_suspend(struct device *dev)
5483 {
5484         struct pci_dev *pdev = to_pci_dev(dev);
5485         int retval;
5486         bool wake;
5487
5488         retval = __e1000_shutdown(pdev, &wake, false);
5489         if (!retval)
5490                 e1000_complete_shutdown(pdev, true, wake);
5491
5492         return retval;
5493 }
5494
5495 static int e1000_resume(struct device *dev)
5496 {
5497         struct pci_dev *pdev = to_pci_dev(dev);
5498         struct net_device *netdev = pci_get_drvdata(pdev);
5499         struct e1000_adapter *adapter = netdev_priv(netdev);
5500
5501         if (e1000e_pm_ready(adapter))
5502                 adapter->idle_check = true;
5503
5504         return __e1000_resume(pdev);
5505 }
5506 #endif /* CONFIG_PM_SLEEP */
5507
5508 #ifdef CONFIG_PM_RUNTIME
5509 static int e1000_runtime_suspend(struct device *dev)
5510 {
5511         struct pci_dev *pdev = to_pci_dev(dev);
5512         struct net_device *netdev = pci_get_drvdata(pdev);
5513         struct e1000_adapter *adapter = netdev_priv(netdev);
5514
5515         if (e1000e_pm_ready(adapter)) {
5516                 bool wake;
5517
5518                 __e1000_shutdown(pdev, &wake, true);
5519         }
5520
5521         return 0;
5522 }
5523
5524 static int e1000_idle(struct device *dev)
5525 {
5526         struct pci_dev *pdev = to_pci_dev(dev);
5527         struct net_device *netdev = pci_get_drvdata(pdev);
5528         struct e1000_adapter *adapter = netdev_priv(netdev);
5529
5530         if (!e1000e_pm_ready(adapter))
5531                 return 0;
5532
5533         if (adapter->idle_check) {
5534                 adapter->idle_check = false;
5535                 if (!e1000e_has_link(adapter))
5536                         pm_schedule_suspend(dev, MSEC_PER_SEC);
5537         }
5538
5539         return -EBUSY;
5540 }
5541
5542 static int e1000_runtime_resume(struct device *dev)
5543 {
5544         struct pci_dev *pdev = to_pci_dev(dev);
5545         struct net_device *netdev = pci_get_drvdata(pdev);
5546         struct e1000_adapter *adapter = netdev_priv(netdev);
5547
5548         if (!e1000e_pm_ready(adapter))
5549                 return 0;
5550
5551         adapter->idle_check = !dev->power.runtime_auto;
5552         return __e1000_resume(pdev);
5553 }
5554 #endif /* CONFIG_PM_RUNTIME */
5555 #endif /* CONFIG_PM */
5556
5557 static void e1000_shutdown(struct pci_dev *pdev)
5558 {
5559         bool wake = false;
5560
5561         __e1000_shutdown(pdev, &wake, false);
5562
5563         if (system_state == SYSTEM_POWER_OFF)
5564                 e1000_complete_shutdown(pdev, false, wake);
5565 }
5566
5567 #ifdef CONFIG_NET_POLL_CONTROLLER
5568
5569 static irqreturn_t e1000_intr_msix(int irq, void *data)
5570 {
5571         struct net_device *netdev = data;
5572         struct e1000_adapter *adapter = netdev_priv(netdev);
5573
5574         if (adapter->msix_entries) {
5575                 int vector, msix_irq;
5576
5577                 vector = 0;
5578                 msix_irq = adapter->msix_entries[vector].vector;
5579                 disable_irq(msix_irq);
5580                 e1000_intr_msix_rx(msix_irq, netdev);
5581                 enable_irq(msix_irq);
5582
5583                 vector++;
5584                 msix_irq = adapter->msix_entries[vector].vector;
5585                 disable_irq(msix_irq);
5586                 e1000_intr_msix_tx(msix_irq, netdev);
5587                 enable_irq(msix_irq);
5588
5589                 vector++;
5590                 msix_irq = adapter->msix_entries[vector].vector;
5591                 disable_irq(msix_irq);
5592                 e1000_msix_other(msix_irq, netdev);
5593                 enable_irq(msix_irq);
5594         }
5595
5596         return IRQ_HANDLED;
5597 }
5598
5599 /*
5600  * Polling 'interrupt' - used by things like netconsole to send skbs
5601  * without having to re-enable interrupts. It's not called while
5602  * the interrupt routine is executing.
5603  */
5604 static void e1000_netpoll(struct net_device *netdev)
5605 {
5606         struct e1000_adapter *adapter = netdev_priv(netdev);
5607
5608         switch (adapter->int_mode) {
5609         case E1000E_INT_MODE_MSIX:
5610                 e1000_intr_msix(adapter->pdev->irq, netdev);
5611                 break;
5612         case E1000E_INT_MODE_MSI:
5613                 disable_irq(adapter->pdev->irq);
5614                 e1000_intr_msi(adapter->pdev->irq, netdev);
5615                 enable_irq(adapter->pdev->irq);
5616                 break;
5617         default: /* E1000E_INT_MODE_LEGACY */
5618                 disable_irq(adapter->pdev->irq);
5619                 e1000_intr(adapter->pdev->irq, netdev);
5620                 enable_irq(adapter->pdev->irq);
5621                 break;
5622         }
5623 }
5624 #endif
5625
5626 /**
5627  * e1000_io_error_detected - called when PCI error is detected
5628  * @pdev: Pointer to PCI device
5629  * @state: The current pci connection state
5630  *
5631  * This function is called after a PCI bus error affecting
5632  * this device has been detected.
5633  */
5634 static pci_ers_result_t e1000_io_error_detected(struct pci_dev *pdev,
5635                                                 pci_channel_state_t state)
5636 {
5637         struct net_device *netdev = pci_get_drvdata(pdev);
5638         struct e1000_adapter *adapter = netdev_priv(netdev);
5639
5640         netif_device_detach(netdev);
5641
5642         if (state == pci_channel_io_perm_failure)
5643                 return PCI_ERS_RESULT_DISCONNECT;
5644
5645         if (netif_running(netdev))
5646                 e1000e_down(adapter);
5647         pci_disable_device(pdev);
5648
5649         /* Request a slot slot reset. */
5650         return PCI_ERS_RESULT_NEED_RESET;
5651 }
5652
5653 /**
5654  * e1000_io_slot_reset - called after the pci bus has been reset.
5655  * @pdev: Pointer to PCI device
5656  *
5657  * Restart the card from scratch, as if from a cold-boot. Implementation
5658  * resembles the first-half of the e1000_resume routine.
5659  */
5660 static pci_ers_result_t e1000_io_slot_reset(struct pci_dev *pdev)
5661 {
5662         struct net_device *netdev = pci_get_drvdata(pdev);
5663         struct e1000_adapter *adapter = netdev_priv(netdev);
5664         struct e1000_hw *hw = &adapter->hw;
5665         u16 aspm_disable_flag = 0;
5666         int err;
5667         pci_ers_result_t result;
5668
5669         if (adapter->flags2 & FLAG2_DISABLE_ASPM_L0S)
5670                 aspm_disable_flag = PCIE_LINK_STATE_L0S;
5671         if (adapter->flags2 & FLAG2_DISABLE_ASPM_L1)
5672                 aspm_disable_flag |= PCIE_LINK_STATE_L1;
5673         if (aspm_disable_flag)
5674                 e1000e_disable_aspm(pdev, aspm_disable_flag);
5675
5676         err = pci_enable_device_mem(pdev);
5677         if (err) {
5678                 dev_err(&pdev->dev,
5679                         "Cannot re-enable PCI device after reset.\n");
5680                 result = PCI_ERS_RESULT_DISCONNECT;
5681         } else {
5682                 pci_set_master(pdev);
5683                 pdev->state_saved = true;
5684                 pci_restore_state(pdev);
5685
5686                 pci_enable_wake(pdev, PCI_D3hot, 0);
5687                 pci_enable_wake(pdev, PCI_D3cold, 0);
5688
5689                 e1000e_reset(adapter);
5690                 ew32(WUS, ~0);
5691                 result = PCI_ERS_RESULT_RECOVERED;
5692         }
5693
5694         pci_cleanup_aer_uncorrect_error_status(pdev);
5695
5696         return result;
5697 }
5698
5699 /**
5700  * e1000_io_resume - called when traffic can start flowing again.
5701  * @pdev: Pointer to PCI device
5702  *
5703  * This callback is called when the error recovery driver tells us that
5704  * its OK to resume normal operation. Implementation resembles the
5705  * second-half of the e1000_resume routine.
5706  */
5707 static void e1000_io_resume(struct pci_dev *pdev)
5708 {
5709         struct net_device *netdev = pci_get_drvdata(pdev);
5710         struct e1000_adapter *adapter = netdev_priv(netdev);
5711
5712         e1000_init_manageability_pt(adapter);
5713
5714         if (netif_running(netdev)) {
5715                 if (e1000e_up(adapter)) {
5716                         dev_err(&pdev->dev,
5717                                 "can't bring device back up after reset\n");
5718                         return;
5719                 }
5720         }
5721
5722         netif_device_attach(netdev);
5723
5724         /*
5725          * If the controller has AMT, do not set DRV_LOAD until the interface
5726          * is up.  For all other cases, let the f/w know that the h/w is now
5727          * under the control of the driver.
5728          */
5729         if (!(adapter->flags & FLAG_HAS_AMT))
5730                 e1000e_get_hw_control(adapter);
5731
5732 }
5733
5734 static void e1000_print_device_info(struct e1000_adapter *adapter)
5735 {
5736         struct e1000_hw *hw = &adapter->hw;
5737         struct net_device *netdev = adapter->netdev;
5738         u32 ret_val;
5739         u8 pba_str[E1000_PBANUM_LENGTH];
5740
5741         /* print bus type/speed/width info */
5742         e_info("(PCI Express:2.5GT/s:%s) %pM\n",
5743                /* bus width */
5744                ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
5745                 "Width x1"),
5746                /* MAC address */
5747                netdev->dev_addr);
5748         e_info("Intel(R) PRO/%s Network Connection\n",
5749                (hw->phy.type == e1000_phy_ife) ? "10/100" : "1000");
5750         ret_val = e1000_read_pba_string_generic(hw, pba_str,
5751                                                 E1000_PBANUM_LENGTH);
5752         if (ret_val)
5753                 strncpy((char *)pba_str, "Unknown", sizeof(pba_str) - 1);
5754         e_info("MAC: %d, PHY: %d, PBA No: %s\n",
5755                hw->mac.type, hw->phy.type, pba_str);
5756 }
5757
5758 static void e1000_eeprom_checks(struct e1000_adapter *adapter)
5759 {
5760         struct e1000_hw *hw = &adapter->hw;
5761         int ret_val;
5762         u16 buf = 0;
5763
5764         if (hw->mac.type != e1000_82573)
5765                 return;
5766
5767         ret_val = e1000_read_nvm(hw, NVM_INIT_CONTROL2_REG, 1, &buf);
5768         if (!ret_val && (!(le16_to_cpu(buf) & (1 << 0)))) {
5769                 /* Deep Smart Power Down (DSPD) */
5770                 dev_warn(&adapter->pdev->dev,
5771                          "Warning: detected DSPD enabled in EEPROM\n");
5772         }
5773 }
5774
5775 static const struct net_device_ops e1000e_netdev_ops = {
5776         .ndo_open               = e1000_open,
5777         .ndo_stop               = e1000_close,
5778         .ndo_start_xmit         = e1000_xmit_frame,
5779         .ndo_get_stats64        = e1000e_get_stats64,
5780         .ndo_set_multicast_list = e1000_set_multi,
5781         .ndo_set_mac_address    = e1000_set_mac,
5782         .ndo_change_mtu         = e1000_change_mtu,
5783         .ndo_do_ioctl           = e1000_ioctl,
5784         .ndo_tx_timeout         = e1000_tx_timeout,
5785         .ndo_validate_addr      = eth_validate_addr,
5786
5787         .ndo_vlan_rx_add_vid    = e1000_vlan_rx_add_vid,
5788         .ndo_vlan_rx_kill_vid   = e1000_vlan_rx_kill_vid,
5789 #ifdef CONFIG_NET_POLL_CONTROLLER
5790         .ndo_poll_controller    = e1000_netpoll,
5791 #endif
5792 };
5793
5794 /**
5795  * e1000_probe - Device Initialization Routine
5796  * @pdev: PCI device information struct
5797  * @ent: entry in e1000_pci_tbl
5798  *
5799  * Returns 0 on success, negative on failure
5800  *
5801  * e1000_probe initializes an adapter identified by a pci_dev structure.
5802  * The OS initialization, configuring of the adapter private structure,
5803  * and a hardware reset occur.
5804  **/
5805 static int __devinit e1000_probe(struct pci_dev *pdev,
5806                                  const struct pci_device_id *ent)
5807 {
5808         struct net_device *netdev;
5809         struct e1000_adapter *adapter;
5810         struct e1000_hw *hw;
5811         const struct e1000_info *ei = e1000_info_tbl[ent->driver_data];
5812         resource_size_t mmio_start, mmio_len;
5813         resource_size_t flash_start, flash_len;
5814
5815         static int cards_found;
5816         u16 aspm_disable_flag = 0;
5817         int i, err, pci_using_dac;
5818         u16 eeprom_data = 0;
5819         u16 eeprom_apme_mask = E1000_EEPROM_APME;
5820
5821         if (ei->flags2 & FLAG2_DISABLE_ASPM_L0S)
5822                 aspm_disable_flag = PCIE_LINK_STATE_L0S;
5823         if (ei->flags2 & FLAG2_DISABLE_ASPM_L1)
5824                 aspm_disable_flag |= PCIE_LINK_STATE_L1;
5825         if (aspm_disable_flag)
5826                 e1000e_disable_aspm(pdev, aspm_disable_flag);
5827
5828         err = pci_enable_device_mem(pdev);
5829         if (err)
5830                 return err;
5831
5832         pci_using_dac = 0;
5833         err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
5834         if (!err) {
5835                 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
5836                 if (!err)
5837                         pci_using_dac = 1;
5838         } else {
5839                 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
5840                 if (err) {
5841                         err = dma_set_coherent_mask(&pdev->dev,
5842                                                     DMA_BIT_MASK(32));
5843                         if (err) {
5844                                 dev_err(&pdev->dev, "No usable DMA "
5845                                         "configuration, aborting\n");
5846                                 goto err_dma;
5847                         }
5848                 }
5849         }
5850
5851         err = pci_request_selected_regions_exclusive(pdev,
5852                                           pci_select_bars(pdev, IORESOURCE_MEM),
5853                                           e1000e_driver_name);
5854         if (err)
5855                 goto err_pci_reg;
5856
5857         /* AER (Advanced Error Reporting) hooks */
5858         pci_enable_pcie_error_reporting(pdev);
5859
5860         pci_set_master(pdev);
5861         /* PCI config space info */
5862         err = pci_save_state(pdev);
5863         if (err)
5864                 goto err_alloc_etherdev;
5865
5866         err = -ENOMEM;
5867         netdev = alloc_etherdev(sizeof(struct e1000_adapter));
5868         if (!netdev)
5869                 goto err_alloc_etherdev;
5870
5871         SET_NETDEV_DEV(netdev, &pdev->dev);
5872
5873         netdev->irq = pdev->irq;
5874
5875         pci_set_drvdata(pdev, netdev);
5876         adapter = netdev_priv(netdev);
5877         hw = &adapter->hw;
5878         adapter->netdev = netdev;
5879         adapter->pdev = pdev;
5880         adapter->ei = ei;
5881         adapter->pba = ei->pba;
5882         adapter->flags = ei->flags;
5883         adapter->flags2 = ei->flags2;
5884         adapter->hw.adapter = adapter;
5885         adapter->hw.mac.type = ei->mac;
5886         adapter->max_hw_frame_size = ei->max_hw_frame_size;
5887         adapter->msg_enable = (1 << NETIF_MSG_DRV | NETIF_MSG_PROBE) - 1;
5888
5889         mmio_start = pci_resource_start(pdev, 0);
5890         mmio_len = pci_resource_len(pdev, 0);
5891
5892         err = -EIO;
5893         adapter->hw.hw_addr = ioremap(mmio_start, mmio_len);
5894         if (!adapter->hw.hw_addr)
5895                 goto err_ioremap;
5896
5897         if ((adapter->flags & FLAG_HAS_FLASH) &&
5898             (pci_resource_flags(pdev, 1) & IORESOURCE_MEM)) {
5899                 flash_start = pci_resource_start(pdev, 1);
5900                 flash_len = pci_resource_len(pdev, 1);
5901                 adapter->hw.flash_address = ioremap(flash_start, flash_len);
5902                 if (!adapter->hw.flash_address)
5903                         goto err_flashmap;
5904         }
5905
5906         /* construct the net_device struct */
5907         netdev->netdev_ops              = &e1000e_netdev_ops;
5908         e1000e_set_ethtool_ops(netdev);
5909         netdev->watchdog_timeo          = 5 * HZ;
5910         netif_napi_add(netdev, &adapter->napi, e1000_clean, 64);
5911         strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
5912
5913         netdev->mem_start = mmio_start;
5914         netdev->mem_end = mmio_start + mmio_len;
5915
5916         adapter->bd_number = cards_found++;
5917
5918         e1000e_check_options(adapter);
5919
5920         /* setup adapter struct */
5921         err = e1000_sw_init(adapter);
5922         if (err)
5923                 goto err_sw_init;
5924
5925         memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
5926         memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
5927         memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
5928
5929         err = ei->get_variants(adapter);
5930         if (err)
5931                 goto err_hw_init;
5932
5933         if ((adapter->flags & FLAG_IS_ICH) &&
5934             (adapter->flags & FLAG_READ_ONLY_NVM))
5935                 e1000e_write_protect_nvm_ich8lan(&adapter->hw);
5936
5937         hw->mac.ops.get_bus_info(&adapter->hw);
5938
5939         adapter->hw.phy.autoneg_wait_to_complete = 0;
5940
5941         /* Copper options */
5942         if (adapter->hw.phy.media_type == e1000_media_type_copper) {
5943                 adapter->hw.phy.mdix = AUTO_ALL_MODES;
5944                 adapter->hw.phy.disable_polarity_correction = 0;
5945                 adapter->hw.phy.ms_type = e1000_ms_hw_default;
5946         }
5947
5948         if (e1000_check_reset_block(&adapter->hw))
5949                 e_info("PHY reset is blocked due to SOL/IDER session.\n");
5950
5951         netdev->features = NETIF_F_SG |
5952                            NETIF_F_HW_CSUM |
5953                            NETIF_F_HW_VLAN_TX |
5954                            NETIF_F_HW_VLAN_RX;
5955
5956         if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER)
5957                 netdev->features |= NETIF_F_HW_VLAN_FILTER;
5958
5959         netdev->features |= NETIF_F_TSO;
5960         netdev->features |= NETIF_F_TSO6;
5961
5962         netdev->vlan_features |= NETIF_F_TSO;
5963         netdev->vlan_features |= NETIF_F_TSO6;
5964         netdev->vlan_features |= NETIF_F_HW_CSUM;
5965         netdev->vlan_features |= NETIF_F_SG;
5966
5967         if (pci_using_dac) {
5968                 netdev->features |= NETIF_F_HIGHDMA;
5969                 netdev->vlan_features |= NETIF_F_HIGHDMA;
5970         }
5971
5972         if (e1000e_enable_mng_pass_thru(&adapter->hw))
5973                 adapter->flags |= FLAG_MNG_PT_ENABLED;
5974
5975         /*
5976          * before reading the NVM, reset the controller to
5977          * put the device in a known good starting state
5978          */
5979         adapter->hw.mac.ops.reset_hw(&adapter->hw);
5980
5981         /*
5982          * systems with ASPM and others may see the checksum fail on the first
5983          * attempt. Let's give it a few tries
5984          */
5985         for (i = 0;; i++) {
5986                 if (e1000_validate_nvm_checksum(&adapter->hw) >= 0)
5987                         break;
5988                 if (i == 2) {
5989                         e_err("The NVM Checksum Is Not Valid\n");
5990                         err = -EIO;
5991                         goto err_eeprom;
5992                 }
5993         }
5994
5995         e1000_eeprom_checks(adapter);
5996
5997         /* copy the MAC address */
5998         if (e1000e_read_mac_addr(&adapter->hw))
5999                 e_err("NVM Read Error while reading MAC address\n");
6000
6001         memcpy(netdev->dev_addr, adapter->hw.mac.addr, netdev->addr_len);
6002         memcpy(netdev->perm_addr, adapter->hw.mac.addr, netdev->addr_len);
6003
6004         if (!is_valid_ether_addr(netdev->perm_addr)) {
6005                 e_err("Invalid MAC Address: %pM\n", netdev->perm_addr);
6006                 err = -EIO;
6007                 goto err_eeprom;
6008         }
6009
6010         init_timer(&adapter->watchdog_timer);
6011         adapter->watchdog_timer.function = e1000_watchdog;
6012         adapter->watchdog_timer.data = (unsigned long) adapter;
6013
6014         init_timer(&adapter->phy_info_timer);
6015         adapter->phy_info_timer.function = e1000_update_phy_info;
6016         adapter->phy_info_timer.data = (unsigned long) adapter;
6017
6018         INIT_WORK(&adapter->reset_task, e1000_reset_task);
6019         INIT_WORK(&adapter->watchdog_task, e1000_watchdog_task);
6020         INIT_WORK(&adapter->downshift_task, e1000e_downshift_workaround);
6021         INIT_WORK(&adapter->update_phy_task, e1000e_update_phy_task);
6022         INIT_WORK(&adapter->print_hang_task, e1000_print_hw_hang);
6023
6024         /* Initialize link parameters. User can change them with ethtool */
6025         adapter->hw.mac.autoneg = 1;
6026         adapter->fc_autoneg = 1;
6027         adapter->hw.fc.requested_mode = e1000_fc_default;
6028         adapter->hw.fc.current_mode = e1000_fc_default;
6029         adapter->hw.phy.autoneg_advertised = 0x2f;
6030
6031         /* ring size defaults */
6032         adapter->rx_ring->count = 256;
6033         adapter->tx_ring->count = 256;
6034
6035         /*
6036          * Initial Wake on LAN setting - If APM wake is enabled in
6037          * the EEPROM, enable the ACPI Magic Packet filter
6038          */
6039         if (adapter->flags & FLAG_APME_IN_WUC) {
6040                 /* APME bit in EEPROM is mapped to WUC.APME */
6041                 eeprom_data = er32(WUC);
6042                 eeprom_apme_mask = E1000_WUC_APME;
6043                 if ((hw->mac.type > e1000_ich10lan) &&
6044                     (eeprom_data & E1000_WUC_PHY_WAKE))
6045                         adapter->flags2 |= FLAG2_HAS_PHY_WAKEUP;
6046         } else if (adapter->flags & FLAG_APME_IN_CTRL3) {
6047                 if (adapter->flags & FLAG_APME_CHECK_PORT_B &&
6048                     (adapter->hw.bus.func == 1))
6049                         e1000_read_nvm(&adapter->hw,
6050                                 NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
6051                 else
6052                         e1000_read_nvm(&adapter->hw,
6053                                 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
6054         }
6055
6056         /* fetch WoL from EEPROM */
6057         if (eeprom_data & eeprom_apme_mask)
6058                 adapter->eeprom_wol |= E1000_WUFC_MAG;
6059
6060         /*
6061          * now that we have the eeprom settings, apply the special cases
6062          * where the eeprom may be wrong or the board simply won't support
6063          * wake on lan on a particular port
6064          */
6065         if (!(adapter->flags & FLAG_HAS_WOL))
6066                 adapter->eeprom_wol = 0;
6067
6068         /* initialize the wol settings based on the eeprom settings */
6069         adapter->wol = adapter->eeprom_wol;
6070         device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
6071
6072         /* save off EEPROM version number */
6073         e1000_read_nvm(&adapter->hw, 5, 1, &adapter->eeprom_vers);
6074
6075         /* reset the hardware with the new settings */
6076         e1000e_reset(adapter);
6077
6078         /*
6079          * If the controller has AMT, do not set DRV_LOAD until the interface
6080          * is up.  For all other cases, let the f/w know that the h/w is now
6081          * under the control of the driver.
6082          */
6083         if (!(adapter->flags & FLAG_HAS_AMT))
6084                 e1000e_get_hw_control(adapter);
6085
6086         strncpy(netdev->name, "eth%d", sizeof(netdev->name) - 1);
6087         err = register_netdev(netdev);
6088         if (err)
6089                 goto err_register;
6090
6091         /* carrier off reporting is important to ethtool even BEFORE open */
6092         netif_carrier_off(netdev);
6093
6094         e1000_print_device_info(adapter);
6095
6096         if (pci_dev_run_wake(pdev))
6097                 pm_runtime_put_noidle(&pdev->dev);
6098
6099         return 0;
6100
6101 err_register:
6102         if (!(adapter->flags & FLAG_HAS_AMT))
6103                 e1000e_release_hw_control(adapter);
6104 err_eeprom:
6105         if (!e1000_check_reset_block(&adapter->hw))
6106                 e1000_phy_hw_reset(&adapter->hw);
6107 err_hw_init:
6108         kfree(adapter->tx_ring);
6109         kfree(adapter->rx_ring);
6110 err_sw_init:
6111         if (adapter->hw.flash_address)
6112                 iounmap(adapter->hw.flash_address);
6113         e1000e_reset_interrupt_capability(adapter);
6114 err_flashmap:
6115         iounmap(adapter->hw.hw_addr);
6116 err_ioremap:
6117         free_netdev(netdev);
6118 err_alloc_etherdev:
6119         pci_release_selected_regions(pdev,
6120                                      pci_select_bars(pdev, IORESOURCE_MEM));
6121 err_pci_reg:
6122 err_dma:
6123         pci_disable_device(pdev);
6124         return err;
6125 }
6126
6127 /**
6128  * e1000_remove - Device Removal Routine
6129  * @pdev: PCI device information struct
6130  *
6131  * e1000_remove is called by the PCI subsystem to alert the driver
6132  * that it should release a PCI device.  The could be caused by a
6133  * Hot-Plug event, or because the driver is going to be removed from
6134  * memory.
6135  **/
6136 static void __devexit e1000_remove(struct pci_dev *pdev)
6137 {
6138         struct net_device *netdev = pci_get_drvdata(pdev);
6139         struct e1000_adapter *adapter = netdev_priv(netdev);
6140         bool down = test_bit(__E1000_DOWN, &adapter->state);
6141
6142         /*
6143          * The timers may be rescheduled, so explicitly disable them
6144          * from being rescheduled.
6145          */
6146         if (!down)
6147                 set_bit(__E1000_DOWN, &adapter->state);
6148         del_timer_sync(&adapter->watchdog_timer);
6149         del_timer_sync(&adapter->phy_info_timer);
6150
6151         cancel_work_sync(&adapter->reset_task);
6152         cancel_work_sync(&adapter->watchdog_task);
6153         cancel_work_sync(&adapter->downshift_task);
6154         cancel_work_sync(&adapter->update_phy_task);
6155         cancel_work_sync(&adapter->print_hang_task);
6156
6157         if (!(netdev->flags & IFF_UP))
6158                 e1000_power_down_phy(adapter);
6159
6160         /* Don't lie to e1000_close() down the road. */
6161         if (!down)
6162                 clear_bit(__E1000_DOWN, &adapter->state);
6163         unregister_netdev(netdev);
6164
6165         if (pci_dev_run_wake(pdev))
6166                 pm_runtime_get_noresume(&pdev->dev);
6167
6168         /*
6169          * Release control of h/w to f/w.  If f/w is AMT enabled, this
6170          * would have already happened in close and is redundant.
6171          */
6172         e1000e_release_hw_control(adapter);
6173
6174         e1000e_reset_interrupt_capability(adapter);
6175         kfree(adapter->tx_ring);
6176         kfree(adapter->rx_ring);
6177
6178         iounmap(adapter->hw.hw_addr);
6179         if (adapter->hw.flash_address)
6180                 iounmap(adapter->hw.flash_address);
6181         pci_release_selected_regions(pdev,
6182                                      pci_select_bars(pdev, IORESOURCE_MEM));
6183
6184         free_netdev(netdev);
6185
6186         /* AER disable */
6187         pci_disable_pcie_error_reporting(pdev);
6188
6189         pci_disable_device(pdev);
6190 }
6191
6192 /* PCI Error Recovery (ERS) */
6193 static struct pci_error_handlers e1000_err_handler = {
6194         .error_detected = e1000_io_error_detected,
6195         .slot_reset = e1000_io_slot_reset,
6196         .resume = e1000_io_resume,
6197 };
6198
6199 static DEFINE_PCI_DEVICE_TABLE(e1000_pci_tbl) = {
6200         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_COPPER), board_82571 },
6201         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_FIBER), board_82571 },
6202         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_QUAD_COPPER), board_82571 },
6203         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_QUAD_COPPER_LP), board_82571 },
6204         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_QUAD_FIBER), board_82571 },
6205         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_SERDES), board_82571 },
6206         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_SERDES_DUAL), board_82571 },
6207         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_SERDES_QUAD), board_82571 },
6208         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571PT_QUAD_COPPER), board_82571 },
6209
6210         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI), board_82572 },
6211         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI_COPPER), board_82572 },
6212         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI_FIBER), board_82572 },
6213         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI_SERDES), board_82572 },
6214
6215         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82573E), board_82573 },
6216         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82573E_IAMT), board_82573 },
6217         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82573L), board_82573 },
6218
6219         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82574L), board_82574 },
6220         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82574LA), board_82574 },
6221         { PCI_VDEVICE(INTEL, E1000_DEV_ID_82583V), board_82583 },
6222
6223         { PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_COPPER_DPT),
6224           board_80003es2lan },
6225         { PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_COPPER_SPT),
6226           board_80003es2lan },
6227         { PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_SERDES_DPT),
6228           board_80003es2lan },
6229         { PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_SERDES_SPT),
6230           board_80003es2lan },
6231
6232         { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IFE), board_ich8lan },
6233         { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IFE_G), board_ich8lan },
6234         { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IFE_GT), board_ich8lan },
6235         { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_AMT), board_ich8lan },
6236         { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_C), board_ich8lan },
6237         { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_M), board_ich8lan },
6238         { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_M_AMT), board_ich8lan },
6239         { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_82567V_3), board_ich8lan },
6240
6241         { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IFE), board_ich9lan },
6242         { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IFE_G), board_ich9lan },
6243         { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IFE_GT), board_ich9lan },
6244         { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_AMT), board_ich9lan },
6245         { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_C), board_ich9lan },
6246         { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_BM), board_ich9lan },
6247         { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_M), board_ich9lan },
6248         { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_M_AMT), board_ich9lan },
6249         { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_M_V), board_ich9lan },
6250
6251         { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_R_BM_LM), board_ich9lan },
6252         { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_R_BM_LF), board_ich9lan },
6253         { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_R_BM_V), board_ich9lan },
6254
6255         { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_D_BM_LM), board_ich10lan },
6256         { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_D_BM_LF), board_ich10lan },
6257         { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_D_BM_V), board_ich10lan },
6258
6259         { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_M_HV_LM), board_pchlan },
6260         { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_M_HV_LC), board_pchlan },
6261         { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_D_HV_DM), board_pchlan },
6262         { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_D_HV_DC), board_pchlan },
6263
6264         { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH2_LV_LM), board_pch2lan },
6265         { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH2_LV_V), board_pch2lan },
6266
6267         { }     /* terminate list */
6268 };
6269 MODULE_DEVICE_TABLE(pci, e1000_pci_tbl);
6270
6271 #ifdef CONFIG_PM
6272 static const struct dev_pm_ops e1000_pm_ops = {
6273         SET_SYSTEM_SLEEP_PM_OPS(e1000_suspend, e1000_resume)
6274         SET_RUNTIME_PM_OPS(e1000_runtime_suspend,
6275                                 e1000_runtime_resume, e1000_idle)
6276 };
6277 #endif
6278
6279 /* PCI Device API Driver */
6280 static struct pci_driver e1000_driver = {
6281         .name     = e1000e_driver_name,
6282         .id_table = e1000_pci_tbl,
6283         .probe    = e1000_probe,
6284         .remove   = __devexit_p(e1000_remove),
6285 #ifdef CONFIG_PM
6286         .driver.pm = &e1000_pm_ops,
6287 #endif
6288         .shutdown = e1000_shutdown,
6289         .err_handler = &e1000_err_handler
6290 };
6291
6292 /**
6293  * e1000_init_module - Driver Registration Routine
6294  *
6295  * e1000_init_module is the first routine called when the driver is
6296  * loaded. All it does is register with the PCI subsystem.
6297  **/
6298 static int __init e1000_init_module(void)
6299 {
6300         int ret;
6301         pr_info("Intel(R) PRO/1000 Network Driver - %s\n",
6302                 e1000e_driver_version);
6303         pr_info("Copyright(c) 1999 - 2011 Intel Corporation.\n");
6304         ret = pci_register_driver(&e1000_driver);
6305
6306         return ret;
6307 }
6308 module_init(e1000_init_module);
6309
6310 /**
6311  * e1000_exit_module - Driver Exit Cleanup Routine
6312  *
6313  * e1000_exit_module is called just before the driver is removed
6314  * from memory.
6315  **/
6316 static void __exit e1000_exit_module(void)
6317 {
6318         pci_unregister_driver(&e1000_driver);
6319 }
6320 module_exit(e1000_exit_module);
6321
6322
6323 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
6324 MODULE_DESCRIPTION("Intel(R) PRO/1000 Network Driver");
6325 MODULE_LICENSE("GPL");
6326 MODULE_VERSION(DRV_VERSION);
6327
6328 /* e1000_main.c */